From b191320ecc1400aa44cb57a2ff08ea8b4e0d0c2b Mon Sep 17 00:00:00 2001 From: Benjamin Diedrichsen Date: Tue, 24 Jun 2014 21:09:49 +0200 Subject: [PATCH] changed report API and chart generator --- .../lab/{Experiment.java => Benchmark.java} | 128 ++++++++----- .../net/engio/pips/lab/ExecutionContext.java | 14 +- .../java/net/engio/pips/lab/LabException.java | 32 ++++ .../java/net/engio/pips/lab/Laboratory.java | 176 ++++-------------- .../net/engio/pips/lab/WorkloadManager.java | 169 +++++++++++++++++ .../engio/pips/lab/{ => common}/Range.java | 2 +- .../pips/lab/{ => common}/ValueGenerator.java | 2 +- .../net/engio/pips/lab/workload/Workload.java | 52 +++++- .../engio/pips/reports/CSVFileExporter.java | 10 +- .../engio/pips/reports/ChartGenerator.java | 53 +++--- .../net/engio/pips/reports/IReporter.java | 4 +- .../net/engio/pips/reports/SeriesGroup.java | 74 +++----- .../net/engio/lab/ExecutionContextTest.java | 10 +- .../java/net/engio/lab/LaboratoryTest.java | 157 ++++++++++++++-- 14 files changed, 590 insertions(+), 293 deletions(-) rename src/main/java/net/engio/pips/lab/{Experiment.java => Benchmark.java} (52%) create mode 100644 src/main/java/net/engio/pips/lab/LabException.java create mode 100644 src/main/java/net/engio/pips/lab/WorkloadManager.java rename src/main/java/net/engio/pips/lab/{ => common}/Range.java (98%) rename src/main/java/net/engio/pips/lab/{ => common}/ValueGenerator.java (76%) diff --git a/src/main/java/net/engio/pips/lab/Experiment.java b/src/main/java/net/engio/pips/lab/Benchmark.java similarity index 52% rename from src/main/java/net/engio/pips/lab/Experiment.java rename to src/main/java/net/engio/pips/lab/Benchmark.java index b4e12c4..c7c9e9c 100644 --- a/src/main/java/net/engio/pips/lab/Experiment.java +++ b/src/main/java/net/engio/pips/lab/Benchmark.java @@ -1,5 +1,6 @@ package net.engio.pips.lab; +import net.engio.pips.data.DataCollectorManager; import net.engio.pips.data.IDataCollector; import net.engio.pips.lab.workload.Workload; import net.engio.pips.reports.IReporter; @@ -7,10 +8,7 @@ import net.engio.pips.reports.IReporter; import java.io.File; import java.io.OutputStream; import java.io.PrintWriter; -import java.util.Collection; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; /** * A benchmark is the container for all information of a formerly executed performance @@ -20,7 +18,7 @@ import java.util.Map; * @author bennidi * Date: 2/11/14 */ -public class Experiment { +public class Benchmark { public static final class Properties{ public static final String TimeoutInSeconds = "Timeout in seconds"; @@ -29,11 +27,9 @@ public class Experiment { public static final String LogStream = "Log stream"; public static final String Title = "Title"; public static final String ReportBaseDir = "Report base dir"; - public static final String Collectors = "collectors:"; - public static final String ExecutionTimers = Collectors + "execution-timer:"; } - private ExecutionContext context = new ExecutionContext(this); + private ExecutionContext rootContext = new ExecutionContext(this); private List reporters = new LinkedList(); @@ -43,12 +39,18 @@ public class Experiment { private String title; - public Experiment(String title) { + private DataCollectorManager collectors = new DataCollectorManager(); + + public Benchmark(String title) { if (title == null || title.isEmpty()) throw new IllegalArgumentException("Please provide a title that is a valid identifier for a directory"); this.title = title; } + public IDataCollector addCollector(IDataCollector collector){ + return collectors.addCollector(collector); + } + public void setExecutions(Executions executions) { this.executions = executions; } @@ -57,12 +59,65 @@ public class Experiment { return executions; } - public Collection getCollectors(){ - return executions.getMatching(Properties.Collectors); + public DataCollectorManager getCollectorManager(){ + return collectors; } - public Collection getCollectors(String collectorId){ - return executions.getMatching(Experiment.Properties.Collectors + collectorId); + public List getCollectors(){ + return getCollectors(""); + } + + public List getCollectors(String collectorId){ + return collectors.getCollectors(collectorId); + } + + void verifyWorkloads(){ + // TOdo: check start/duration dependencies + + for(Workload workload : getWorkloads()){ + if(workload.getITaskFactory() == null) + throw new LabException("Workload has no task factory:" + workload, LabException.ErrorCode.WLWithoutFactory); + if(workload.getStartCondition() == null) + throw new LabException("Workload has no start condition specified:" + workload, LabException.ErrorCode.WLWithoutStart); + if(workload.getDuration() == null) + throw new LabException("Workload has no duration specified:" + workload, LabException.ErrorCode.WLWithoutDuration); + } + + // verify dependency graph + for(Workload workload : getWorkloads()){ + // since there is always a finite number of workloads + // traversing the links will either end in cycle or terminate with a workload that specifies an absolute start/duration + if(!willStart(workload)) + throw new LabException("Cycle in workload start condition" + workload, LabException.ErrorCode.WLWithCycleInStart); + if(!willEnd(workload)) + throw new LabException("Cycle in workload duration: " + workload, LabException.ErrorCode.WLWithCycleInDuration); + } + + } + + + private boolean willStart(Workload workload){ + if(workload.getStartCondition().isImmediately() || workload.getStartCondition().isTimebased()) + return true; + Set preceeding = new HashSet(); + while(workload.getStartCondition().getPreceedingWorkload() != null){ + if(!preceeding.add(workload.getStartCondition().getPreceedingWorkload())) + return false; // workload was reached before -> cycle + else workload = workload.getStartCondition().getPreceedingWorkload(); + } + return true; + } + + private boolean willEnd(Workload workload){ + if(workload.getDuration().isRepetitive() || workload.getDuration().isTimeBased()) + return true; + Set preceeding = new HashSet(); + while(workload.getDuration().getDependingOn() != null){ + if(!preceeding.add(workload.getDuration().getDependingOn())) + return false; + else workload = workload.getDuration().getDependingOn(); + } + return true; } @@ -74,27 +129,29 @@ public class Experiment { * @param value - The value to associate with the key * @return */ - public Experiment register(String key, Object value) { - context.bind(key, value); + public Benchmark setProperty(String key, Object value) { + rootContext.bind(key, value); return this; } - public Experiment addWorkload(Workload... workload) { - for (Workload wl : workload) - workloads.add(wl); + public Benchmark addWorkload(Workload... workload) { + for (Workload wl : workload){ + if(wl.hasTasksToRun())workloads.add(wl); + //else getLogStream() // TODO: log warning + } return this; } - public Experiment addReport(IReporter reporter) { + public Benchmark addReporter(IReporter reporter) { reporters.add(reporter); return this; } - public void generateReports() throws Exception { + public void generateReports(IReporter ...reporters) throws Exception { PrintWriter log = new PrintWriter(getLogStream(), true); - if (reporters.isEmpty()) { + if (reporters.length == 0) { log.println("Skipping report generation because no reporters have been registered"); return; } @@ -117,31 +174,23 @@ public class Experiment { return baseDir.getAbsolutePath() + File.separator; } - public T get(String key) { - return context.get(key); - } - public List getWorkloads() { return workloads; } public boolean isDefined(String key) { - return context.containsKey(key); + return rootContext.containsKey(key); } public T getProperty(String key) { - return (T) context.get(key); + return (T) rootContext.get(key); } - public int getSampleInterval() { - return getProperty(Properties.SampleInterval); - } - - public Experiment setBasePath(String basePath) { + public Benchmark setBasePath(String basePath) { return setProperty(Properties.BasePath, basePath); } - public Experiment setSampleInterval(int sampleInterval) { + public Benchmark setSampleInterval(int sampleInterval) { return setProperty(Properties.SampleInterval, sampleInterval); } @@ -149,17 +198,12 @@ public class Experiment { return getProperty(Properties.TimeoutInSeconds); } - public Experiment setProperty(String key, Object value) { - context.bind(key, value); - return this; - } - public OutputStream getLogStream() { return isDefined(Properties.LogStream) ? (OutputStream) getProperty(Properties.LogStream) : System.out; } - public Experiment setLogStream(OutputStream out) { + public Benchmark setLogStream(OutputStream out) { return setProperty(Properties.LogStream, out); } @@ -168,7 +212,7 @@ public class Experiment { StringBuilder exp = new StringBuilder(); exp.append("Experiment "); exp.append(title); - exp.append("with "); + exp.append(" with "); exp.append(workloads.size() + " workloads"); exp.append("\n"); @@ -178,7 +222,7 @@ public class Experiment { } exp.append("\n"); exp.append("and additional parameters:\n"); - for(Map.Entry entry : context.getProperties().entrySet()){ + for(Map.Entry entry : rootContext.getProperties().entrySet()){ exp.append("\t"); exp.append(entry.getKey()); exp.append(":"); @@ -196,7 +240,7 @@ public class Experiment { public ExecutionContext getClobalContext() { - return context; + return rootContext; } } diff --git a/src/main/java/net/engio/pips/lab/ExecutionContext.java b/src/main/java/net/engio/pips/lab/ExecutionContext.java index 46d2d95..33af5ef 100644 --- a/src/main/java/net/engio/pips/lab/ExecutionContext.java +++ b/src/main/java/net/engio/pips/lab/ExecutionContext.java @@ -15,14 +15,14 @@ import java.util.*; */ public class ExecutionContext { - private Experiment experiment; + private Benchmark benchmark; private ExecutionContext parent; private Map properties = new HashMap(); private long started; private long finished; - public ExecutionContext(Experiment experiment) { - this.experiment = experiment; + public ExecutionContext(Benchmark benchmark) { + this.benchmark = benchmark; } public void started(){ @@ -43,15 +43,15 @@ public class ExecutionContext { public ExecutionTimer createExecutionTimer(String timerId){ DataCollector timings = createLocalCollector(timerId); - Sampler sampler = Sampler.timeBased((Integer)get(Experiment.Properties.SampleInterval)); - sampler.pipeInto(timings); + Sampler sampler = Sampler.timeBased((Integer)get(Benchmark.Properties.SampleInterval)); + sampler.connectTo(timings); ExecutionTimer timer = new ExecutionTimer(sampler); return timer; } public DataCollector createLocalCollector(String collectorId){ DataCollector collector = new DataCollector(collectorId); - bind(Experiment.Properties.ExecutionTimers + collectorId, collector); + bind(collectorId, collector); return collector; } @@ -78,7 +78,7 @@ public class ExecutionContext { public ExecutionContext getChild(){ - ExecutionContext child = new ExecutionContext(experiment); + ExecutionContext child = new ExecutionContext(benchmark); child.parent = this; return child; } diff --git a/src/main/java/net/engio/pips/lab/LabException.java b/src/main/java/net/engio/pips/lab/LabException.java new file mode 100644 index 0000000..4f2ceba --- /dev/null +++ b/src/main/java/net/engio/pips/lab/LabException.java @@ -0,0 +1,32 @@ +package net.engio.pips.lab; + +/** + * @author bennidi + * Date: 3/25/14 + */ +public class LabException extends RuntimeException { + + private ErrorCode code; + + public LabException(String message, ErrorCode code) { + super(message); + this.code = code; + } + + public LabException(String message, Throwable cause, ErrorCode code) { + super(message, cause); + this.code = code; + } + + public ErrorCode getCode() { + return code; + } + + public static enum ErrorCode{ + WLWithoutFactory, + WLWithCycleInDuration, + WLWithCycleInStart, + WLWithoutStart, + WLWithoutDuration + } +} diff --git a/src/main/java/net/engio/pips/lab/Laboratory.java b/src/main/java/net/engio/pips/lab/Laboratory.java index 0d3f564..4484efe 100644 --- a/src/main/java/net/engio/pips/lab/Laboratory.java +++ b/src/main/java/net/engio/pips/lab/Laboratory.java @@ -1,13 +1,14 @@ package net.engio.pips.lab; -import net.engio.pips.lab.workload.*; +import net.engio.pips.lab.workload.ExecutionEvent; +import net.engio.pips.lab.workload.ExecutionHandler; +import net.engio.pips.lab.workload.Workload; import java.io.PrintWriter; import java.util.*; -import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.atomic.AtomicInteger; /** @@ -16,34 +17,51 @@ import java.util.concurrent.atomic.AtomicInteger; */ public class Laboratory { - public void run(Experiment... experiments) throws Exception { - for(Experiment experiment : experiments){ - // TODO: verify workload configuration (at least one timebased/immediate wl) - measure(experiment); - PrintWriter log = new PrintWriter(experiment.getLogStream(), true); - log.println("Generating reports...."); - experiment.generateReports(); + public void run(Benchmark... benchmarks) throws Exception { + for(Benchmark benchmark : benchmarks){ + benchmark.verifyWorkloads(); } + for(Benchmark benchmark : benchmarks){ + measure(benchmark); + /* + PrintWriter log = new PrintWriter(benchmark.getLogStream(), true); + log.println("Generating reports...."); + benchmark.generateReports(); */ + } + + Thread.sleep(3000); // wait for shutdown } - public void measure(final Experiment experiment) { + + + public void measure(final Benchmark benchmark) { // each workload will run in its own thread - final ExecutorService executor = Executors.newFixedThreadPool(experiment.getWorkloads().size()); + final ExecutorService executor = Executors.newFixedThreadPool(benchmark.getWorkloads().size(), new ThreadFactory() { + + private ThreadGroup group = new ThreadGroup("scheduler"); + + @Override + public Thread newThread(Runnable runnable) { + Thread thread = new Thread(group, runnable, "Workload scheduler"); + thread.setPriority(Thread.MIN_PRIORITY); + return thread; + } + }); // keeping track of workloads and their corresponding executables - final Map workloads = new HashMap(experiment.getWorkloads().size()); + final Map workloads = new HashMap(benchmark.getWorkloads().size()); //final Map> scheduled = Collections.synchronizedMap(new HashMap>(experiment.getWorkloads().size())); final AtomicInteger finishedWorkloads = new AtomicInteger(0); - final PrintWriter log = new PrintWriter(experiment.getLogStream(), true); + final PrintWriter log = new PrintWriter(benchmark.getLogStream(), true); final Timer timer = new Timer(true); Date start = new Date(System.currentTimeMillis()); log.println("Starting experiment at " + start ); // prepare workloads - for(final Workload workload : experiment.getWorkloads()){ - workloads.put(workload, new WorkloadManager(workload, experiment)); + for(final Workload workload : benchmark.getWorkloads()){ + workloads.put(workload, new WorkloadManager(workload, benchmark)); // keep track of finished workloads workload.handle(ExecutionEvent.WorkloadCompletion, new ExecutionHandler() { @@ -93,7 +111,7 @@ public class Laboratory { } // schedule workloads - for(final Workload workload : experiment.getWorkloads()){ + for(final Workload workload : benchmark.getWorkloads()){ // either now if(workload.getStartCondition().isImmediately()){ workloads.get(workload).start(executor); @@ -111,7 +129,7 @@ public class Laboratory { // wait until all tasks have been executed try { - while(finishedWorkloads.get() < experiment.getWorkloads().size()) + while(finishedWorkloads.get() < benchmark.getWorkloads().size()) Thread.sleep(1000); } catch (InterruptedException e) { e.printStackTrace(); @@ -122,130 +140,12 @@ public class Laboratory { // merge contexts Executions executions = new Executions(); for(WorkloadManager workMan : workloads.values()) - executions.addAll(workMan.contexts); - experiment.setExecutions(executions); + executions.addAll(workMan.getContexts()); + benchmark.setExecutions(executions); } } - - private static class WorkloadManager{ - - private Workload workload; - private Callable scheduler; - private ExecutorService workloadExecutor; - private List scheduledTasks = new LinkedList(); - private Future scheduledWorkload; - private List contexts = new LinkedList(); - private volatile boolean stopped = false; - - private WorkloadManager(Workload workload, Experiment experiment) { - this.workload = workload; - createScheduler(experiment, experiment.getClobalContext().getChild()); - } - - private void stop(){ - stopped = true; - for(Future task : scheduledTasks) - task.cancel(true); // this doesn't seem to have any effect - System.out.println("Canceling workload" + workload); - scheduledWorkload.cancel(true); - workloadExecutor.shutdown(); - } - - private Future start(ExecutorService executor){ - return scheduledWorkload = executor.submit(scheduler); - } - - // create a single executable unit which will run the tasks from the given workload - // in its own thread pool - private Callable createScheduler(final Experiment experiment, final ExecutionContext workloadContext){ - workloadExecutor = Executors.newFixedThreadPool(workload.getParallelUnits()); - scheduler = new Callable() { - @Override - public Long call() { - final AtomicInteger scheduled = new AtomicInteger(0);// number of scheduled tasks - final AtomicInteger finished = new AtomicInteger(0); // number of finished tasks - //final ResultCollector collector = experiment.getResults(); - final ITaskFactory tasks = workload.getITaskFactory(); - final PrintWriter log = new PrintWriter(experiment.getLogStream(), true); - - log.println("Starting workload " + workload.getName()); - // call initialization handlers before scheduling the actual tasks - workload.started(); - workload.getHandler(ExecutionEvent.WorkloadInitialization).handle(workloadContext); - // create the tasks and schedule for execution - for (int i = 0; i < workload.getParallelUnits() ; i++) { - log.println("Scheduling unit " + scheduled.incrementAndGet()); - final int taskNumber = i+1; - final ExecutionContext taskContext = workloadContext.getChild(); - contexts.add(taskContext); - // simply submit a runnable as return values are not important - scheduledTasks.add(workloadExecutor.submit(new Runnable() { - @Override - public void run() { - try { - ITask task = tasks.create(taskContext); - log.println("Executing task " + taskNumber); - if (workload.getDuration().isRepetitive()) { - for (int i = 0; i < workload.getDuration().getRepetitions(); i++) - task.run(taskContext); - } else { - while (!stopped) { - task.run(taskContext); - } - } - } catch (Exception e) { - log.println("Task" + taskNumber + " threw an exception will orderly execution: " + e.toString()); - e.printStackTrace(); - throw new RuntimeException(e); - } finally { - finished.incrementAndGet(); - log.println("Finished task: " + taskNumber); - log.println("Tasks left:" + (scheduled.get() - finished.get())); - } - } - })); - } - - // wait until all tasks have been executed - try { - while(scheduled.get() > finished.get()) - Thread.sleep(1000); - } catch (InterruptedException e) { - if(workload.getDuration().isDependent() && !workload.getDuration().getDependingOn().isFinished()){ - log.println(workload + " interrupted although dependent workload not finished"); - e.printStackTrace(); // something was wrong here - } - - if(!workload.getDuration().isTimeBased() - && !workload.getDuration().isDependent()){ - log.println(workload + " interrupted although no time based duration specified"); - e.printStackTrace(); // something was wrong here - } - if(workload.getDuration().isTimeBased() - // interrupted before duration ends - && System.currentTimeMillis() < workload.getDuration().inMillisecs() + workload.getStarted()){ - log.println(workload + " interrupted before timer finished"); - e.printStackTrace(); // something was wrong here - } - - }finally { - // signal end - workload.finished(); - log.println("Finished workload: " + workload); - workload.getHandler(ExecutionEvent.WorkloadCompletion).handle(workloadContext); - } - return 1L; - } - - }; - return scheduler; - } - - - } - } diff --git a/src/main/java/net/engio/pips/lab/WorkloadManager.java b/src/main/java/net/engio/pips/lab/WorkloadManager.java new file mode 100644 index 0000000..985d920 --- /dev/null +++ b/src/main/java/net/engio/pips/lab/WorkloadManager.java @@ -0,0 +1,169 @@ +package net.engio.pips.lab; + +import net.engio.pips.lab.workload.ExecutionEvent; +import net.engio.pips.lab.workload.ITask; +import net.engio.pips.lab.workload.ITaskFactory; +import net.engio.pips.lab.workload.Workload; + +import java.io.PrintWriter; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Todo: Add javadoc + * + * @author bennidi + * Date: 6/19/14 + */ +class WorkloadManager { + + private Workload workload; + private Callable scheduler; + private ExecutorService workloadExecutor; + private List scheduledTasks = new LinkedList(); + private Future scheduledWorkload; + private List contexts = new LinkedList(); + private volatile boolean stopped = false; + + WorkloadManager(Workload workload, Benchmark benchmark) { + this.workload = workload; + createScheduler(benchmark, benchmark.getClobalContext().getChild()); + } + + void stop() { + stopped = true; + for (Future task : scheduledTasks) + task.cancel(true); // this doesn't seem to have any effect + System.out.println("Canceling workload " + workload.getName()); + scheduledWorkload.cancel(true); + workloadExecutor.shutdown(); + } + + Future start(ExecutorService executor) { + return scheduledWorkload = executor.submit(scheduler); + } + + List getContexts() { + return contexts; + } + + // create a single executable unit which will run the tasks from the given workload + // in its own thread pool + private Callable createScheduler(final Benchmark benchmark, final ExecutionContext workloadContext) { + workloadExecutor = Executors.newFixedThreadPool(workload.getParallelUnits(), new ThreadFactory() { + + private ThreadGroup group = new ThreadGroup(workload.getName()); + @Override + public Thread newThread(Runnable runnable) { + Thread thread = new Thread(group, runnable, workload.getName()); + thread.setPriority(Thread.NORM_PRIORITY); + return thread; + } + }); + scheduler = new Callable() { + @Override + public Long call() { + final AtomicInteger scheduled = new AtomicInteger(0);// number of scheduled tasks + final AtomicInteger finished = new AtomicInteger(0); // number of finished tasks + //final ResultCollector collector = experiment.getResults(); + final ITaskFactory tasks = workload.getITaskFactory(); + final PrintWriter log = new PrintWriter(benchmark.getLogStream(), true); + + log.println("Starting workload " + workload); + // call initialization handlers before scheduling the actual tasks + workload.started(); + workload.getHandler(ExecutionEvent.WorkloadInitialization).handle(workloadContext); + // create the tasks and schedule for execution + for (int i = 0; i < workload.getParallelUnits(); i++) { + log.println("Scheduling task " + workload.getName() + "[" + scheduled.incrementAndGet() + "]"); + final int taskNumber = i + 1; + final ExecutionContext taskContext = workloadContext.getChild(); + contexts.add(taskContext); + // simply submit a runnable as return values are not important + // the runnable creates a new task and keeps executing it according to specified duration + scheduledTasks.add(workloadExecutor.submit(new Runnable() { + @Override + public void run() { + try { + ITask task = tasks.create(taskContext); + log.println("Executing task " + workload.getName() + "[" + taskNumber + "]"); + int round = 0; + // execute number of times specified + if (workload.getDuration().isRepetitive()) { + for (int i = 0; i < workload.getDuration().getRepetitions(); i++) { + execute(task, workload, taskContext, log, taskNumber, ++round); + } + + } else { // or as long as depending task has not yet finished + while (!stopped) { + execute(task, workload, taskContext, log, taskNumber, ++round); + } + } + } catch(InterruptedException e){ + // this happens when the workload is shutdown + Thread.currentThread().interrupt(); + } catch (Exception e) { + log.println("Task" + workload.getName() + "[" + taskNumber + "]" + " threw an exception while orderly execution: " + e.toString()); + e.printStackTrace(); + //throw new RuntimeException(e); + } finally { + finished.incrementAndGet(); + log.println("Finished task: " + workload.getName() + "[" + taskNumber + "]"); + log.println("Tasks left in " + workload.getName() + ": " + (scheduled.get() - finished.get())); + } + } + })); + } + + // wait until all tasks have been executed + try { + while (scheduled.get() > finished.get()) + Thread.sleep(1000); + } catch (InterruptedException e) { + if (workload.getDuration().isDependent() && !workload.getDuration().getDependingOn().isFinished()) { + log.println(workload + " interrupted although dependent workload not finished"); + e.printStackTrace(); // something was wrong here + } + + if (!workload.getDuration().isTimeBased() + && !workload.getDuration().isDependent()) { + log.println(workload + " interrupted although no time based duration specified"); + e.printStackTrace(); // something was wrong here + } + if (workload.getDuration().isTimeBased() + // interrupted before duration ends + && System.currentTimeMillis() < workload.getDuration().inMillisecs() + workload.getStarted()) { + log.println(workload + " interrupted before timer finished"); + e.printStackTrace(); // something was wrong here + } + + } finally { + // signal end + workload.finished(); + log.println("Finished workload: " + workload); + workload.getHandler(ExecutionEvent.WorkloadCompletion).handle(workloadContext); + } + return 1L; + } + + }; + return scheduler; + } + + + private void execute(ITask task, Workload workload, ExecutionContext taskContext, PrintWriter log, int taskNumber, int round) throws InterruptedException { + try { + log.println(workload.getName() + "[" + taskNumber + "]->" + round); + task.run(taskContext); + } catch (Exception e) { + log.println("Task" + workload.getName() + "[" + taskNumber + "]" + " threw an exception while orderly execution: " + e.toString()); + e.printStackTrace(); + //throw new RuntimeException(e); + } + if (workload.hasDelay()) + Thread.sleep(workload.getDelay()); + } + +} diff --git a/src/main/java/net/engio/pips/lab/Range.java b/src/main/java/net/engio/pips/lab/common/Range.java similarity index 98% rename from src/main/java/net/engio/pips/lab/Range.java rename to src/main/java/net/engio/pips/lab/common/Range.java index e69cf39..72d23cd 100644 --- a/src/main/java/net/engio/pips/lab/Range.java +++ b/src/main/java/net/engio/pips/lab/common/Range.java @@ -1,4 +1,4 @@ -package net.engio.pips.lab; +package net.engio.pips.lab.common; import java.util.Arrays; import java.util.Random; diff --git a/src/main/java/net/engio/pips/lab/ValueGenerator.java b/src/main/java/net/engio/pips/lab/common/ValueGenerator.java similarity index 76% rename from src/main/java/net/engio/pips/lab/ValueGenerator.java rename to src/main/java/net/engio/pips/lab/common/ValueGenerator.java index 600cb70..89c1377 100644 --- a/src/main/java/net/engio/pips/lab/ValueGenerator.java +++ b/src/main/java/net/engio/pips/lab/common/ValueGenerator.java @@ -1,4 +1,4 @@ -package net.engio.pips.lab; +package net.engio.pips.lab.common; /** * diff --git a/src/main/java/net/engio/pips/lab/workload/Workload.java b/src/main/java/net/engio/pips/lab/workload/Workload.java index 3d9151b..945a0d0 100644 --- a/src/main/java/net/engio/pips/lab/workload/Workload.java +++ b/src/main/java/net/engio/pips/lab/workload/Workload.java @@ -9,8 +9,8 @@ import java.util.Map; import java.util.concurrent.TimeUnit; /** - * A single workload defines a set of {@link ITask} to be executed as part of an {@link net.engio.pips.lab.Experiment}. - * Tasks are potentially run in parallel depending on the configuration of {@code setParallelTasks}. + * A single workload defines a set of {@link ITask} to be executed as part of an {@link net.engio.pips.lab.Benchmark}. + * Multiple tasks are run in parallel (see {@code setParallelTasks}). * Tasks are created using the corresponding {@link net.engio.pips.lab.workload.ITaskFactory}. * * @@ -25,7 +25,7 @@ public class Workload { private Duration duration; - private StartCondition starting = new StartCondition(); + private StartCondition starting; private String name; @@ -33,6 +33,8 @@ public class Workload { private volatile long finished; + private long delay = -1; + private Map handlers = new HashMap(); @@ -52,6 +54,15 @@ public class Workload { return finished != -1; } + public Workload setDelay(long ms){ + delay = ms; + return this; + } + + public boolean hasDelay(){ + return delay > 0; + } + public long getExecutionTime(){ return isFinished() ? finished - started : -1; } @@ -60,8 +71,19 @@ public class Workload { return parallelUnits; } + public boolean hasTasksToRun(){ + return parallelUnits > 0; + } + + /** + * Define how many task should run in parallel. Tasks are created using the + * specified {@link ITaskFactory} + * + * @param parallelUnits + * @return + */ public Workload setParallelTasks(int parallelUnits) { - if(parallelUnits < 1 )throw new IllegalArgumentException("At least one task must run"); + //if(parallelUnits < 1 )throw new IllegalArgumentException("At least one task must run"); this.parallelUnits = parallelUnits; return this; } @@ -74,12 +96,25 @@ public class Workload { return ITaskFactory; } + /** + * Set the task factory that will be used to create the single tasks of this workload. + * + * @param ITaskFactory The task factory to be used for task creation + * @return This workload + */ public Workload setITaskFactory(ITaskFactory ITaskFactory) { this.ITaskFactory = ITaskFactory; return this; } - + /** + * Add an event handler to this workload. Depending on the type of event + * the handler will be called automatically by the {@link net.engio.pips.lab.Laboratory} + * + * @param event The type of event + * @param handler The handler to be invoked when the event occurs + * @return This workload + */ public Workload handle(ExecutionEvent event, ExecutionHandler handler){ if(handlers.containsKey(event)){ handlers.get(event).delegate.add(handler); @@ -134,6 +169,11 @@ public class Workload { return started; } + public long getDelay() { + return delay; + } + + // intermediate class for clean API public class StartSpecification{ public Workload after(int timeout, TimeUnit unit){ @@ -153,6 +193,7 @@ public class Workload { } + // intermediate class for clean API public class DurationSpecification{ public Workload lasts(int timeout, TimeUnit unit){ @@ -172,6 +213,7 @@ public class Workload { } + // wrap multiple execution handlers public static class ExecutionHandlerWrapper implements ExecutionHandler{ private List delegate = new LinkedList(); diff --git a/src/main/java/net/engio/pips/reports/CSVFileExporter.java b/src/main/java/net/engio/pips/reports/CSVFileExporter.java index 1d705b6..8fee1de 100644 --- a/src/main/java/net/engio/pips/reports/CSVFileExporter.java +++ b/src/main/java/net/engio/pips/reports/CSVFileExporter.java @@ -1,7 +1,7 @@ package net.engio.pips.reports; import net.engio.pips.data.IDataCollector; -import net.engio.pips.lab.Experiment; +import net.engio.pips.lab.Benchmark; import java.io.File; import java.io.PrintWriter; @@ -12,20 +12,20 @@ import java.io.PrintWriter; */ public class CSVFileExporter implements IReporter { - public void generate(Experiment experiment) throws Exception { - String reportDirectory = experiment.getReportBaseDir(); + public void generate(Benchmark benchmark) throws Exception { + String reportDirectory = benchmark.getReportBaseDir(); File report = new File(reportDirectory + "report.txt"); PrintWriter writer = new PrintWriter(report); try { // write report header writer.println("###### EXPERIMENT ##########"); - writer.println(experiment); + writer.println(benchmark); // write data of collectors writer.println(); writer.println("##### COLLECTORS ########"); - for (IDataCollector collector: experiment.getCollectors()) { + for (IDataCollector collector: benchmark.getCollectors()) { writer.println(collector); } diff --git a/src/main/java/net/engio/pips/reports/ChartGenerator.java b/src/main/java/net/engio/pips/reports/ChartGenerator.java index 75e9945..9be4ad7 100644 --- a/src/main/java/net/engio/pips/reports/ChartGenerator.java +++ b/src/main/java/net/engio/pips/reports/ChartGenerator.java @@ -1,20 +1,20 @@ package net.engio.pips.reports; -import net.engio.pips.lab.Experiment; +import net.engio.pips.lab.Benchmark; import org.jfree.chart.ChartFactory; import org.jfree.chart.ChartUtilities; import org.jfree.chart.JFreeChart; +import org.jfree.chart.axis.AxisLocation; import org.jfree.chart.axis.NumberAxis; import org.jfree.chart.plot.XYPlot; +import org.jfree.chart.renderer.xy.XYLineAndShapeRenderer; import org.jfree.data.time.TimeSeriesCollection; import java.awt.*; import java.io.File; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; +import java.util.*; import java.util.List; -import java.util.Map; /** * @@ -50,29 +50,16 @@ public class ChartGenerator implements IReporter { } - /** - * Configure a new group that will be treated as a single dataset in the chart. - * Each dataset has its own range axis. - * @param collectorId : The id used to retrieve associated data collectors from the result collector - * @param groupLabel : The label used for the range axis - * @return - */ - public SeriesGroup addGroup(String groupLabel, String collectorId){ - SeriesGroup g = new SeriesGroup(groupLabel, collectorId); - groups.add(g); - return g; - } - - public ChartGenerator add(SeriesGroup seriesGroup) { + public ChartGenerator draw(SeriesGroup seriesGroup) { groups.add(seriesGroup); return this; } - public void generate(Experiment experiment){ + public void generate(Benchmark benchmark){ Map groupAxis = new HashMap(); // process default group SeriesGroup defaultGroup = this.groups.get(0); - TimeSeriesCollection collection = defaultGroup.createDataSet(experiment); + TimeSeriesCollection collection = defaultGroup.createDataSet(benchmark); int maxNumberOfDatapoints = defaultGroup.getSize(); groupAxis.put(defaultGroup.getLabel(), 0); @@ -94,20 +81,25 @@ public class ChartGenerator implements IReporter { plot.setDomainGridlinesVisible(true); plot.setDomainGridlinePaint(Color.BLACK); plot.setBackgroundPaint(Color.DARK_GRAY); - + plot.setRenderer(0, getRandomRenderer()); // add other groups List groups = this.groups.subList(1, this.groups.size()); int axisIndex = 1, dataSetIndex = 1; for(SeriesGroup group : groups){ - plot.setDataset(dataSetIndex, group.createDataSet(experiment)); + plot.setDataset(dataSetIndex, group.createDataSet(benchmark)); + plot.setRenderer(dataSetIndex, getRandomRenderer()); + // if the group does not share a range axis with an already mapped group if(!groupAxis.containsKey(group.getLabel())){ final NumberAxis axis2 = new NumberAxis(group.getLabel()); // prevent the axis to be scaled from zero if the dataset begins with higher values - axis2.setAutoRangeIncludesZero(false); + axis2.setAutoRangeIncludesZero(true); plot.setRangeAxis(axisIndex, axis2); plot.mapDatasetToRangeAxis(dataSetIndex, axisIndex); + plot.setRangeAxisLocation(axisIndex, group.getOrientation() == SeriesGroup.Orientation.Left + ? AxisLocation.BOTTOM_OR_LEFT + : AxisLocation.BOTTOM_OR_RIGHT); groupAxis.put(group.getLabel(), axisIndex); // remember this axis for subsequent groups axisIndex++; } @@ -124,12 +116,25 @@ public class ChartGenerator implements IReporter { int width = maxNumberOfDatapoints * pixelPerDatapoint; try { - String path = experiment.getReportBaseDir() + filename; + String path = benchmark.getReportBaseDir() + filename; ChartUtilities.saveChartAsJPEG(new File(path), chart, width <= 1024 ? 1024 : width, 1024); } catch (IOException e) { System.err.println("Problem occurred creating chart."); } } + private Random rand = new Random(); + private Color getRandomColor(){ + return new Color(Math.abs(rand.nextInt()) % 256,// r + Math.abs(rand.nextInt()) % 256, // g + Math.abs(rand.nextInt()) % 256); //b + } + + + private XYLineAndShapeRenderer getRandomRenderer(){ + XYLineAndShapeRenderer renderer = new XYLineAndShapeRenderer(); + renderer.setSeriesPaint(0, getRandomColor()); + return renderer; + } } diff --git a/src/main/java/net/engio/pips/reports/IReporter.java b/src/main/java/net/engio/pips/reports/IReporter.java index b9d9920..2854797 100644 --- a/src/main/java/net/engio/pips/reports/IReporter.java +++ b/src/main/java/net/engio/pips/reports/IReporter.java @@ -1,6 +1,6 @@ package net.engio.pips.reports; -import net.engio.pips.lab.Experiment; +import net.engio.pips.lab.Benchmark; /** * Take the benchmark and create some output. This may be everything from logging @@ -11,5 +11,5 @@ import net.engio.pips.lab.Experiment; */ public interface IReporter { - void generate(Experiment experiment) throws Exception; + void generate(Benchmark benchmark) throws Exception; } diff --git a/src/main/java/net/engio/pips/reports/SeriesGroup.java b/src/main/java/net/engio/pips/reports/SeriesGroup.java index 879eab5..3d5d497 100644 --- a/src/main/java/net/engio/pips/reports/SeriesGroup.java +++ b/src/main/java/net/engio/pips/reports/SeriesGroup.java @@ -1,19 +1,14 @@ package net.engio.pips.reports; -import net.engio.pips.data.DataCollector; import net.engio.pips.data.IDataCollector; -import net.engio.pips.data.IDataProcessor; -import net.engio.pips.data.aggregator.Average; -import net.engio.pips.data.utils.ItemCounter; import net.engio.pips.data.utils.TimeBasedAggregator; -import net.engio.pips.lab.Experiment; +import net.engio.pips.lab.Benchmark; import org.jfree.data.time.TimeSeries; import org.jfree.data.time.TimeSeriesCollection; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; -import java.util.Map; +import java.util.List; /** * A series group is used to configure a set of data collectors for being included in a time series @@ -25,30 +20,44 @@ import java.util.Map; */ public class SeriesGroup { - private String collectorId; + public static enum Orientation{ + Left, Right + } private String label; - private Map aggregators = new HashMap(); - private Collection collectors = new ArrayList(); private int size; private String yAxis = ""; - private int collectorSampleSize = 1; + private Orientation orientation = Orientation.Left; - public SeriesGroup(String collectorId, String label) { - this.collectorId = collectorId; + public SeriesGroup(String label) { this.label = label; } + public SeriesGroup setYAxisOrientation(Orientation orientation){ + this.orientation = orientation; + return this; + } + + public Orientation getOrientation() { + return orientation; + } + public SeriesGroup addCollector(IDataCollector collector){ collectors.add(collector); return this; } + public SeriesGroup addCollectors(List collectors){ + for(IDataCollector collector : collectors) + addCollector(collector); + return this; + } + public String getLabel() { return label; } @@ -57,15 +66,6 @@ public class SeriesGroup { return size; } - public String getCollectorId() { - return collectorId; - } - - public SeriesGroup aggregate(String name, IDataProcessor aggregator){ - aggregators.put(name, aggregator); - return this; - } - public String getyAxis() { return yAxis; } @@ -75,45 +75,21 @@ public class SeriesGroup { return this; } - public SeriesGroup setDrawEveryNthGraph(int factor) { - this.collectorSampleSize = factor; - return this; - } - public TimeSeriesCollection createDataSet(Experiment experiment){ - Collection collectors = experiment.getCollectors(collectorId); - collectors.addAll(this.collectors); + + public TimeSeriesCollection createDataSet(Benchmark benchmark){ TimeSeriesCollection collection = new TimeSeriesCollection(); TimeBasedAggregator aggregator = new TimeBasedAggregator(); // create a series from each data collector - int numberOfCollectors = 1; for(IDataCollector collector : collectors){ // ignore empty data collectors as well as according to sample size if(collector == null || collector.size() == 0)continue; - if(numberOfCollectors % collectorSampleSize != 0){ - numberOfCollectors++; - continue; - } - TimeSeriesConsumer wrapper = new TimeSeriesConsumer(collector.getId()); collector.feed(wrapper); TimeSeries series = wrapper.getSeries(); collection.addSeries(series); - // prepare the time based aggregator - if(!aggregators.isEmpty()) - aggregator.consume(collector); - numberOfCollectors++; + if(size < collector.size())size = collector.size(); } - DataCollector average = aggregator.fold(new Average()); - ItemCounter numberOfDatapoints = new ItemCounter(); - for(Map.Entry aggregation : aggregators.entrySet()){ - TimeSeriesConsumer series = new TimeSeriesConsumer(aggregation.getKey()); - IDataProcessor aggregate = aggregation.getValue(); - aggregate.pipeInto(numberOfDatapoints).pipeInto(series); - average.feed(aggregate); - collection.addSeries(series.getSeries()); - } - size = numberOfDatapoints.getItemCount(); return collection; } } diff --git a/src/test/java/net/engio/lab/ExecutionContextTest.java b/src/test/java/net/engio/lab/ExecutionContextTest.java index 568342d..e46bab8 100644 --- a/src/test/java/net/engio/lab/ExecutionContextTest.java +++ b/src/test/java/net/engio/lab/ExecutionContextTest.java @@ -1,8 +1,8 @@ package net.engio.lab; +import net.engio.pips.lab.Benchmark; import net.engio.pips.lab.ExecutionContext; import net.engio.pips.lab.Executions; -import net.engio.pips.lab.Experiment; import org.junit.Test; import java.util.Collection; @@ -20,12 +20,12 @@ public class ExecutionContextTest extends UnitTest{ Map bindings = new HashMap(); bindings.put(ExecutionContext.class.toString(), ExecutionContext.class); bindings.put(UnitTest.class.toString(), UnitTest.class); - bindings.put(Experiment.class.toString(), Experiment.class); + bindings.put(Benchmark.class.toString(), Benchmark.class); return bindings; } private ExecutionContext getInitialContext(Map bindings){ - ExecutionContext ctx = new ExecutionContext(new Experiment("test")); + ExecutionContext ctx = new ExecutionContext(new Benchmark("test")); ctx.bindAll(bindings); assertBindingsExist(bindings, ctx); @@ -104,7 +104,7 @@ public class ExecutionContextTest extends UnitTest{ @Test public void testGetMatching(){ - ExecutionContext ctx = new ExecutionContext(new Experiment("test")); + ExecutionContext ctx = new ExecutionContext(new Benchmark("test")); Object[] bindings = new Object[]{"root", "root:lvl1", "root:lvl1:lvl2", "none"}; ctx.bind(bindings); @@ -130,7 +130,7 @@ public class ExecutionContextTest extends UnitTest{ @Test public void testExecutions(){ - ExecutionContext ctx = new ExecutionContext(new Experiment("test")); + ExecutionContext ctx = new ExecutionContext(new Benchmark("test")); Object[] bindings = new Object[]{"root", "root:lvl1", "root:lvl1:lvl2", "none"}; ctx.bind(bindings); diff --git a/src/test/java/net/engio/lab/LaboratoryTest.java b/src/test/java/net/engio/lab/LaboratoryTest.java index 3c75803..3fcbe00 100644 --- a/src/test/java/net/engio/lab/LaboratoryTest.java +++ b/src/test/java/net/engio/lab/LaboratoryTest.java @@ -1,7 +1,8 @@ package net.engio.lab; +import net.engio.pips.lab.Benchmark; import net.engio.pips.lab.ExecutionContext; -import net.engio.pips.lab.Experiment; +import net.engio.pips.lab.LabException; import net.engio.pips.lab.Laboratory; import net.engio.pips.lab.workload.*; import org.junit.Test; @@ -29,8 +30,81 @@ public class LaboratoryTest extends UnitTest{ } }; + @Test + public void testInvalidWorkloadSetups() throws Exception { + Workload withoutFactory = new Workload("without factory") + .starts().immediately() + .duration().repetitions(1); + + Workload withoutStart = new Workload("without start") + .setITaskFactory(NoOperation) + .duration().repetitions(1); + + Workload withoutDuration = new Workload("without duration") + .starts().immediately() + .setITaskFactory(NoOperation); + Workload cyclicStart1 = new Workload("without duration"); + Workload cyclicStart2 = new Workload("without duration"); + cyclicStart1 + .starts().after(cyclicStart2) + .duration().repetitions(1) + .setITaskFactory(NoOperation); + cyclicStart2 + .starts().after(cyclicStart1) + .duration().repetitions(1) + .setITaskFactory(NoOperation); + + Workload cyclicDuration1 = new Workload("without duration"); + Workload cyclicDuration2 = new Workload("without duration"); + cyclicDuration1 + .starts().immediately() + .duration().depends(cyclicDuration2) + .setITaskFactory(NoOperation); + cyclicDuration2 + .starts().immediately() + .duration().depends(cyclicDuration1) + .setITaskFactory(NoOperation); + + Laboratory lab = new Laboratory(); + + try { + lab.run(new Benchmark("Invalid").addWorkload(withoutFactory)); + fail(); + } catch (LabException e) { + assertEquals(e.getCode(), LabException.ErrorCode.WLWithoutFactory); + } + + try { + lab.run(new Benchmark("Invalid").addWorkload(withoutStart)); + fail(); + } catch (LabException e) { + assertEquals(e.getCode(), LabException.ErrorCode.WLWithoutStart); + } + + try { + lab.run(new Benchmark("Invalid").addWorkload(withoutDuration)); + fail(); + } catch (LabException e) { + assertEquals(e.getCode(), LabException.ErrorCode.WLWithoutDuration); + } + + try { + lab.run(new Benchmark("Invalid").addWorkload(cyclicStart1, cyclicStart2)); + fail(); + } catch (LabException e) { + assertEquals(e.getCode(), LabException.ErrorCode.WLWithCycleInStart); + } + + try { + lab.run(new Benchmark("Invalid").addWorkload(cyclicDuration1, cyclicDuration2)); + fail(); + } catch (LabException e) { + assertEquals(e.getCode(), LabException.ErrorCode.WLWithCycleInDuration); + } + + } @Test public void testWorkloadSchedulingStartingAfter() throws Exception { @@ -75,17 +149,18 @@ public class LaboratoryTest extends UnitTest{ .starts().after(first); Laboratory lab = new Laboratory(); - lab.run(new Experiment("test").addWorkload(first, second)); + lab.run(new Benchmark("test").addWorkload(first, second)); Thread.sleep(100); // both have run in the end assertEquals(0, counter.get()); } - @Test - public void testWorkloadShutdownCancelsTasks() throws Exception { - final AtomicInteger counter = new AtomicInteger(1); - Workload countUp = new Workload("First workload") + //@Test + // TODO: this invalid start/end dependency tree must be fixed + public void testWorkloadSchedulingStartingAfterDelayed() throws Exception { + final AtomicInteger counter = new AtomicInteger(0); + Workload first = new Workload("First workload") .setParallelTasks(15) .setITaskFactory(new ITaskFactory() { @Override @@ -98,6 +173,58 @@ public class LaboratoryTest extends UnitTest{ }; } }) + .duration().repetitions(1) + .starts().after(2, TimeUnit.SECONDS); + + Workload second = new Workload("Second Workload") + .setParallelTasks(15) + .setITaskFactory(new ITaskFactory() { + @Override + public ITask create(ExecutionContext context) { + return new ITask() { + @Override + public void run(ExecutionContext context) throws Exception { + counter.decrementAndGet(); + } + }; + } + }) + .handle(ExecutionEvent.WorkloadInitialization, new ExecutionHandler() { + @Override + public void handle(ExecutionContext context) { + // the first workload + assertEquals(15, counter.get()); + } + }) + .duration().depends(first) + .starts().after(first); + + Laboratory lab = new Laboratory(); + lab.run(new Benchmark("test").addWorkload(first, second)); + + Thread.sleep(4000); + // both have run in the end + assertEquals(0, counter.get()); + } + + @Test + public void testWorkloadShutdownCancelsTasks() throws Exception { + final AtomicInteger first = new AtomicInteger(1); + final AtomicInteger second = new AtomicInteger(1); + + Workload countUp = new Workload("First workload") + .setParallelTasks(15) + .setITaskFactory(new ITaskFactory() { + @Override + public ITask create(ExecutionContext context) { + return new ITask() { + @Override + public void run(ExecutionContext context) throws Exception { + first.incrementAndGet(); + } + }; + } + }) .duration().lasts(5, TimeUnit.SECONDS) .starts().immediately(); @@ -109,7 +236,7 @@ public class LaboratoryTest extends UnitTest{ return new ITask() { @Override public void run(ExecutionContext context) throws Exception { - counter.decrementAndGet(); + second.incrementAndGet(); } }; } @@ -118,14 +245,16 @@ public class LaboratoryTest extends UnitTest{ .starts().immediately(); Laboratory lab = new Laboratory(); - lab.run(new Experiment("test").addWorkload(countUp, countDown)); + lab.run(new Benchmark("test").addWorkload(countUp, countDown)); - int count = counter.get(); + int firstVal = first.get(); + int secondVal = second.get(); // no threads that change the counter are running anymore Thread.sleep(1000); - assertEquals(count, counter.get()); - // both have run in the end - assertTrue(counter.get() < 16); + assertEquals(firstVal, first.get()); + assertEquals(secondVal, second.get()); + assertTrue(firstVal > 1); + assertTrue(secondVal > 1); } @Test @@ -166,7 +295,7 @@ public class LaboratoryTest extends UnitTest{ .starts().immediately(); Laboratory lab = new Laboratory(); - lab.run(new Experiment("test").addWorkload(first, second)); + lab.run(new Benchmark("test").addWorkload(first, second)); // first workload has run at least duration seconds assertTrue(finish.get() - start.get() >= duration * 1000); @@ -203,7 +332,7 @@ public class LaboratoryTest extends UnitTest{ }); Laboratory lab = new Laboratory(); - lab.run(new Experiment("test").addWorkload(first, second, third)); + lab.run(new Benchmark("test").addWorkload(first, second, third)); assertTrue(finished.get()); }