dorkbox
/
lab
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

changed report API and chart generator

master
Benjamin Diedrichsen 2014-06-24 21:09:49 +02:00
parent 3a0af9e2ee
commit b191320ecc
14 changed files with 590 additions and 293 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

128
src/main/java/net/engio/pips/lab/Experiment.java → src/main/java/net/engio/pips/lab/Benchmark.java
View File

@ -1,5 +1,6 @@
package net.engio.pips.lab;
import net.engio.pips.data.DataCollectorManager;
import net.engio.pips.data.IDataCollector;
import net.engio.pips.lab.workload.Workload;
import net.engio.pips.reports.IReporter;
@ -7,10 +8,7 @@ import net.engio.pips.reports.IReporter;
import java.io.File;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.*;
/**
* A benchmark is the container for all information of a formerly executed performance
@ -20,7 +18,7 @@ import java.util.Map;
* @author bennidi
* Date: 2/11/14
*/
public class Experiment {
public class Benchmark {
public static final class Properties{
public static final String TimeoutInSeconds = "Timeout in seconds";
@ -29,11 +27,9 @@ public class Experiment {
public static final String LogStream = "Log stream";
public static final String Title = "Title";
public static final String ReportBaseDir = "Report base dir";
public static final String Collectors = "collectors:";
public static final String ExecutionTimers = Collectors + "execution-timer:";
}
private ExecutionContext context = new ExecutionContext(this);
private ExecutionContext rootContext = new ExecutionContext(this);
private List<IReporter> reporters = new LinkedList<IReporter>();
@ -43,12 +39,18 @@ public class Experiment {
private String title;
public Experiment(String title) {
private DataCollectorManager collectors = new DataCollectorManager();
public Benchmark(String title) {
if (title == null || title.isEmpty())
throw new IllegalArgumentException("Please provide a title that is a valid identifier for a directory");
this.title = title;
}
public <V> IDataCollector<V> addCollector(IDataCollector<V> collector){
return collectors.addCollector(collector);
}
public void setExecutions(Executions executions) {
this.executions = executions;
}
@ -57,12 +59,65 @@ public class Experiment {
return executions;
}
public Collection<IDataCollector> getCollectors(){
return executions.getMatching(Properties.Collectors);
public DataCollectorManager getCollectorManager(){
return collectors;
}
public Collection<IDataCollector> getCollectors(String collectorId){
return executions.getMatching(Experiment.Properties.Collectors + collectorId);
public List<IDataCollector> getCollectors(){
return getCollectors("");
}
public List<IDataCollector> getCollectors(String collectorId){
return collectors.getCollectors(collectorId);
}
void verifyWorkloads(){
// TOdo: check start/duration dependencies
for(Workload workload : getWorkloads()){
if(workload.getITaskFactory() == null)
throw new LabException("Workload has no task factory:" + workload, LabException.ErrorCode.WLWithoutFactory);
if(workload.getStartCondition() == null)
throw new LabException("Workload has no start condition specified:" + workload, LabException.ErrorCode.WLWithoutStart);
if(workload.getDuration() == null)
throw new LabException("Workload has no duration specified:" + workload, LabException.ErrorCode.WLWithoutDuration);
}
// verify dependency graph
for(Workload workload : getWorkloads()){
// since there is always a finite number of workloads
// traversing the links will either end in cycle or terminate with a workload that specifies an absolute start/duration
if(!willStart(workload))
throw new LabException("Cycle in workload start condition" + workload, LabException.ErrorCode.WLWithCycleInStart);
if(!willEnd(workload))
throw new LabException("Cycle in workload duration: " + workload, LabException.ErrorCode.WLWithCycleInDuration);
}
}
private boolean willStart(Workload workload){
if(workload.getStartCondition().isImmediately() || workload.getStartCondition().isTimebased())
return true;
Set<Workload> preceeding = new HashSet<Workload>();
while(workload.getStartCondition().getPreceedingWorkload() != null){
if(!preceeding.add(workload.getStartCondition().getPreceedingWorkload()))
return false; // workload was reached before -> cycle
else workload = workload.getStartCondition().getPreceedingWorkload();
}
return true;
}
private boolean willEnd(Workload workload){
if(workload.getDuration().isRepetitive() || workload.getDuration().isTimeBased())
return true;
Set<Workload> preceeding = new HashSet<Workload>();
while(workload.getDuration().getDependingOn() != null){
if(!preceeding.add(workload.getDuration().getDependingOn()))
return false;
else workload = workload.getDuration().getDependingOn();
}
return true;
}
@ -74,27 +129,29 @@ public class Experiment {
* @param value - The value to associate with the key
* @return
*/
public Experiment register(String key, Object value) {
context.bind(key, value);
public Benchmark setProperty(String key, Object value) {
rootContext.bind(key, value);
return this;
}
public Experiment addWorkload(Workload... workload) {
for (Workload wl : workload)
workloads.add(wl);
public Benchmark addWorkload(Workload... workload) {
for (Workload wl : workload){
if(wl.hasTasksToRun())workloads.add(wl);
//else getLogStream() // TODO: log warning
}
return this;
}
public Experiment addReport(IReporter reporter) {
public Benchmark addReporter(IReporter reporter) {
reporters.add(reporter);
return this;
}
public void generateReports() throws Exception {
public void generateReports(IReporter ...reporters) throws Exception {
PrintWriter log = new PrintWriter(getLogStream(), true);
if (reporters.isEmpty()) {
if (reporters.length == 0) {
log.println("Skipping report generation because no reporters have been registered");
return;
}
@ -117,31 +174,23 @@ public class Experiment {
return baseDir.getAbsolutePath() + File.separator;
}
public <T> T get(String key) {
return context.get(key);
}
public List<Workload> getWorkloads() {
return workloads;
}
public boolean isDefined(String key) {
return context.containsKey(key);
return rootContext.containsKey(key);
}
public <T> T getProperty(String key) {
return (T) context.get(key);
return (T) rootContext.get(key);
}
public int getSampleInterval() {
return getProperty(Properties.SampleInterval);
}
public Experiment setBasePath(String basePath) {
public Benchmark setBasePath(String basePath) {
return setProperty(Properties.BasePath, basePath);
}
public Experiment setSampleInterval(int sampleInterval) {
public Benchmark setSampleInterval(int sampleInterval) {
return setProperty(Properties.SampleInterval, sampleInterval);
}
@ -149,17 +198,12 @@ public class Experiment {
return getProperty(Properties.TimeoutInSeconds);
}
public Experiment setProperty(String key, Object value) {
context.bind(key, value);
return this;
}
public OutputStream getLogStream() {
return isDefined(Properties.LogStream) ? (OutputStream) getProperty(Properties.LogStream) : System.out;
}
public Experiment setLogStream(OutputStream out) {
public Benchmark setLogStream(OutputStream out) {
return setProperty(Properties.LogStream, out);
}
@ -168,7 +212,7 @@ public class Experiment {
StringBuilder exp = new StringBuilder();
exp.append("Experiment ");
exp.append(title);
exp.append("with ");
exp.append(" with ");
exp.append(workloads.size() + " workloads");
exp.append("\n");
@ -178,7 +222,7 @@ public class Experiment {
}
exp.append("\n");
exp.append("and additional parameters:\n");
for(Map.Entry entry : context.getProperties().entrySet()){
for(Map.Entry entry : rootContext.getProperties().entrySet()){
exp.append("\t");
exp.append(entry.getKey());
exp.append(":");
@ -196,7 +240,7 @@ public class Experiment {
public ExecutionContext getClobalContext() {
return context;
return rootContext;
}
}

14
src/main/java/net/engio/pips/lab/ExecutionContext.java
View File

@ -15,14 +15,14 @@ import java.util.*;
*/
public class ExecutionContext {
private Experiment experiment;
private Benchmark benchmark;
private ExecutionContext parent;
private Map<String, Object> properties = new HashMap<String, Object>();
private long started;
private long finished;
public ExecutionContext(Experiment experiment) {
this.experiment = experiment;
public ExecutionContext(Benchmark benchmark) {
this.benchmark = benchmark;
}
public void started(){
@ -43,15 +43,15 @@ public class ExecutionContext {
public ExecutionTimer createExecutionTimer(String timerId){
DataCollector<Long> timings = createLocalCollector(timerId);
Sampler sampler = Sampler.<Long>timeBased((Integer)get(Experiment.Properties.SampleInterval));
sampler.pipeInto(timings);
Sampler sampler = Sampler.<Long>timeBased((Integer)get(Benchmark.Properties.SampleInterval));
sampler.connectTo(timings);
ExecutionTimer timer = new ExecutionTimer(sampler);
return timer;
}
public <V> DataCollector<V> createLocalCollector(String collectorId){
DataCollector<V> collector = new DataCollector(collectorId);
bind(Experiment.Properties.ExecutionTimers + collectorId, collector);
bind(collectorId, collector);
return collector;
}
@ -78,7 +78,7 @@ public class ExecutionContext {
public ExecutionContext getChild(){
ExecutionContext child = new ExecutionContext(experiment);
ExecutionContext child = new ExecutionContext(benchmark);
child.parent = this;
return child;
}

32
src/main/java/net/engio/pips/lab/LabException.java Normal file
View File

@ -0,0 +1,32 @@
package net.engio.pips.lab;
/**
* @author bennidi
* Date: 3/25/14
*/
public class LabException extends RuntimeException {
private ErrorCode code;
public LabException(String message, ErrorCode code) {
super(message);
this.code = code;
}
public LabException(String message, Throwable cause, ErrorCode code) {
super(message, cause);
this.code = code;
}
public ErrorCode getCode() {
return code;
}
public static enum ErrorCode{
WLWithoutFactory,
WLWithCycleInDuration,
WLWithCycleInStart,
WLWithoutStart,
WLWithoutDuration
}
}

176
src/main/java/net/engio/pips/lab/Laboratory.java
View File

@ -1,13 +1,14 @@
package net.engio.pips.lab;
import net.engio.pips.lab.workload.*;
import net.engio.pips.lab.workload.ExecutionEvent;
import net.engio.pips.lab.workload.ExecutionHandler;
import net.engio.pips.lab.workload.Workload;
import java.io.PrintWriter;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicInteger;
/**
@ -16,34 +17,51 @@ import java.util.concurrent.atomic.AtomicInteger;
*/
public class Laboratory {
public void run(Experiment... experiments) throws Exception {
for(Experiment experiment : experiments){
// TODO: verify workload configuration (at least one timebased/immediate wl)
measure(experiment);
PrintWriter log = new PrintWriter(experiment.getLogStream(), true);
log.println("Generating reports....");
experiment.generateReports();
public void run(Benchmark... benchmarks) throws Exception {
for(Benchmark benchmark : benchmarks){
benchmark.verifyWorkloads();
}
for(Benchmark benchmark : benchmarks){
measure(benchmark);
/*
PrintWriter log = new PrintWriter(benchmark.getLogStream(), true);
log.println("Generating reports....");
benchmark.generateReports(); */
}
Thread.sleep(3000); // wait for shutdown
}
public void measure(final Experiment experiment) {
public void measure(final Benchmark benchmark) {
// each workload will run in its own thread
final ExecutorService executor = Executors.newFixedThreadPool(experiment.getWorkloads().size());
final ExecutorService executor = Executors.newFixedThreadPool(benchmark.getWorkloads().size(), new ThreadFactory() {
private ThreadGroup group = new ThreadGroup("scheduler");
@Override
public Thread newThread(Runnable runnable) {
Thread thread = new Thread(group, runnable, "Workload scheduler");
thread.setPriority(Thread.MIN_PRIORITY);
return thread;
}
});
// keeping track of workloads and their corresponding executables
final Map<Workload, WorkloadManager> workloads = new HashMap<Workload, WorkloadManager>(experiment.getWorkloads().size());
final Map<Workload, WorkloadManager> workloads = new HashMap<Workload, WorkloadManager>(benchmark.getWorkloads().size());
//final Map<Workload, Future<Long>> scheduled = Collections.synchronizedMap(new HashMap<Workload, Future<Long>>(experiment.getWorkloads().size()));
final AtomicInteger finishedWorkloads = new AtomicInteger(0);
final PrintWriter log = new PrintWriter(experiment.getLogStream(), true);
final PrintWriter log = new PrintWriter(benchmark.getLogStream(), true);
final Timer timer = new Timer(true);
Date start = new Date(System.currentTimeMillis());
log.println("Starting experiment at " + start );
// prepare workloads
for(final Workload workload : experiment.getWorkloads()){
workloads.put(workload, new WorkloadManager(workload, experiment));
for(final Workload workload : benchmark.getWorkloads()){
workloads.put(workload, new WorkloadManager(workload, benchmark));
// keep track of finished workloads
workload.handle(ExecutionEvent.WorkloadCompletion, new ExecutionHandler() {
@ -93,7 +111,7 @@ public class Laboratory {
}
// schedule workloads
for(final Workload workload : experiment.getWorkloads()){
for(final Workload workload : benchmark.getWorkloads()){
// either now
if(workload.getStartCondition().isImmediately()){
workloads.get(workload).start(executor);
@ -111,7 +129,7 @@ public class Laboratory {
// wait until all tasks have been executed
try {
while(finishedWorkloads.get() < experiment.getWorkloads().size())
while(finishedWorkloads.get() < benchmark.getWorkloads().size())
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
@ -122,130 +140,12 @@ public class Laboratory {
// merge contexts
Executions executions = new Executions();
for(WorkloadManager workMan : workloads.values())
executions.addAll(workMan.contexts);
experiment.setExecutions(executions);
executions.addAll(workMan.getContexts());
benchmark.setExecutions(executions);
}
}
private static class WorkloadManager{
private Workload workload;
private Callable<Long> scheduler;
private ExecutorService workloadExecutor;
private List<Future> scheduledTasks = new LinkedList<Future>();
private Future scheduledWorkload;
private List<ExecutionContext> contexts = new LinkedList<ExecutionContext>();
private volatile boolean stopped = false;
private WorkloadManager(Workload workload, Experiment experiment) {
this.workload = workload;
createScheduler(experiment, experiment.getClobalContext().getChild());
}
private void stop(){
stopped = true;
for(Future task : scheduledTasks)
task.cancel(true); // this doesn't seem to have any effect
System.out.println("Canceling workload" + workload);
scheduledWorkload.cancel(true);
workloadExecutor.shutdown();
}
private Future start(ExecutorService executor){
return scheduledWorkload = executor.submit(scheduler);
}
// create a single executable unit which will run the tasks from the given workload
// in its own thread pool
private Callable<Long> createScheduler(final Experiment experiment, final ExecutionContext workloadContext){
workloadExecutor = Executors.newFixedThreadPool(workload.getParallelUnits());
scheduler = new Callable<Long>() {
@Override
public Long call() {
final AtomicInteger scheduled = new AtomicInteger(0);// number of scheduled tasks
final AtomicInteger finished = new AtomicInteger(0); // number of finished tasks
//final ResultCollector collector = experiment.getResults();
final ITaskFactory tasks = workload.getITaskFactory();
final PrintWriter log = new PrintWriter(experiment.getLogStream(), true);
log.println("Starting workload " + workload.getName());
// call initialization handlers before scheduling the actual tasks
workload.started();
workload.getHandler(ExecutionEvent.WorkloadInitialization).handle(workloadContext);
// create the tasks and schedule for execution
for (int i = 0; i < workload.getParallelUnits() ; i++) {
log.println("Scheduling unit " + scheduled.incrementAndGet());
final int taskNumber = i+1;
final ExecutionContext taskContext = workloadContext.getChild();
contexts.add(taskContext);
// simply submit a runnable as return values are not important
scheduledTasks.add(workloadExecutor.submit(new Runnable() {
@Override
public void run() {
try {
ITask task = tasks.create(taskContext);
log.println("Executing task " + taskNumber);
if (workload.getDuration().isRepetitive()) {
for (int i = 0; i < workload.getDuration().getRepetitions(); i++)
task.run(taskContext);
} else {
while (!stopped) {
task.run(taskContext);
}
}
} catch (Exception e) {
log.println("Task" + taskNumber + " threw an exception will orderly execution: " + e.toString());
e.printStackTrace();
throw new RuntimeException(e);
} finally {
finished.incrementAndGet();
log.println("Finished task: " + taskNumber);
log.println("Tasks left:" + (scheduled.get() - finished.get()));
}
}
}));
}
// wait until all tasks have been executed
try {
while(scheduled.get() > finished.get())
Thread.sleep(1000);
} catch (InterruptedException e) {
if(workload.getDuration().isDependent() && !workload.getDuration().getDependingOn().isFinished()){
log.println(workload + " interrupted although dependent workload not finished");
e.printStackTrace(); // something was wrong here
}
if(!workload.getDuration().isTimeBased()
&& !workload.getDuration().isDependent()){
log.println(workload + " interrupted although no time based duration specified");
e.printStackTrace(); // something was wrong here
}
if(workload.getDuration().isTimeBased()
// interrupted before duration ends
&& System.currentTimeMillis() < workload.getDuration().inMillisecs() + workload.getStarted()){
log.println(workload + " interrupted before timer finished");
e.printStackTrace(); // something was wrong here
}
}finally {
// signal end
workload.finished();
log.println("Finished workload: " + workload);
workload.getHandler(ExecutionEvent.WorkloadCompletion).handle(workloadContext);
}
return 1L;
}
};
return scheduler;
}
}
}

169
src/main/java/net/engio/pips/lab/WorkloadManager.java Normal file
View File

@ -0,0 +1,169 @@
package net.engio.pips.lab;
import net.engio.pips.lab.workload.ExecutionEvent;
import net.engio.pips.lab.workload.ITask;
import net.engio.pips.lab.workload.ITaskFactory;
import net.engio.pips.lab.workload.Workload;
import java.io.PrintWriter;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Todo: Add javadoc
*
* @author bennidi
* Date: 6/19/14
*/
class WorkloadManager {
private Workload workload;
private Callable<Long> scheduler;
private ExecutorService workloadExecutor;
private List<Future> scheduledTasks = new LinkedList<Future>();
private Future scheduledWorkload;
private List<ExecutionContext> contexts = new LinkedList<ExecutionContext>();
private volatile boolean stopped = false;
WorkloadManager(Workload workload, Benchmark benchmark) {
this.workload = workload;
createScheduler(benchmark, benchmark.getClobalContext().getChild());
}
void stop() {
stopped = true;
for (Future task : scheduledTasks)
task.cancel(true); // this doesn't seem to have any effect
System.out.println("Canceling workload " + workload.getName());
scheduledWorkload.cancel(true);
workloadExecutor.shutdown();
}
Future start(ExecutorService executor) {
return scheduledWorkload = executor.submit(scheduler);
}
List<ExecutionContext> getContexts() {
return contexts;
}
// create a single executable unit which will run the tasks from the given workload
// in its own thread pool
private Callable<Long> createScheduler(final Benchmark benchmark, final ExecutionContext workloadContext) {
workloadExecutor = Executors.newFixedThreadPool(workload.getParallelUnits(), new ThreadFactory() {
private ThreadGroup group = new ThreadGroup(workload.getName());
@Override
public Thread newThread(Runnable runnable) {
Thread thread = new Thread(group, runnable, workload.getName());
thread.setPriority(Thread.NORM_PRIORITY);
return thread;
}
});
scheduler = new Callable<Long>() {
@Override
public Long call() {
final AtomicInteger scheduled = new AtomicInteger(0);// number of scheduled tasks
final AtomicInteger finished = new AtomicInteger(0); // number of finished tasks
//final ResultCollector collector = experiment.getResults();
final ITaskFactory tasks = workload.getITaskFactory();
final PrintWriter log = new PrintWriter(benchmark.getLogStream(), true);
log.println("Starting workload " + workload);
// call initialization handlers before scheduling the actual tasks
workload.started();
workload.getHandler(ExecutionEvent.WorkloadInitialization).handle(workloadContext);
// create the tasks and schedule for execution
for (int i = 0; i < workload.getParallelUnits(); i++) {
log.println("Scheduling task " + workload.getName() + "[" + scheduled.incrementAndGet() + "]");
final int taskNumber = i + 1;
final ExecutionContext taskContext = workloadContext.getChild();
contexts.add(taskContext);
// simply submit a runnable as return values are not important
// the runnable creates a new task and keeps executing it according to specified duration
scheduledTasks.add(workloadExecutor.submit(new Runnable() {
@Override
public void run() {
try {
ITask task = tasks.create(taskContext);
log.println("Executing task " + workload.getName() + "[" + taskNumber + "]");
int round = 0;
// execute number of times specified
if (workload.getDuration().isRepetitive()) {
for (int i = 0; i < workload.getDuration().getRepetitions(); i++) {
execute(task, workload, taskContext, log, taskNumber, ++round);
}
} else { // or as long as depending task has not yet finished
while (!stopped) {
execute(task, workload, taskContext, log, taskNumber, ++round);
}
}
} catch(InterruptedException e){
// this happens when the workload is shutdown
Thread.currentThread().interrupt();
} catch (Exception e) {
log.println("Task" + workload.getName() + "[" + taskNumber + "]" + " threw an exception while orderly execution: " + e.toString());
e.printStackTrace();
//throw new RuntimeException(e);
} finally {
finished.incrementAndGet();
log.println("Finished task: " + workload.getName() + "[" + taskNumber + "]");
log.println("Tasks left in " + workload.getName() + ": " + (scheduled.get() - finished.get()));
}
}
}));
}
// wait until all tasks have been executed
try {
while (scheduled.get() > finished.get())
Thread.sleep(1000);
} catch (InterruptedException e) {
if (workload.getDuration().isDependent() && !workload.getDuration().getDependingOn().isFinished()) {
log.println(workload + " interrupted although dependent workload not finished");
e.printStackTrace(); // something was wrong here
}
if (!workload.getDuration().isTimeBased()
&& !workload.getDuration().isDependent()) {
log.println(workload + " interrupted although no time based duration specified");
e.printStackTrace(); // something was wrong here
}
if (workload.getDuration().isTimeBased()
// interrupted before duration ends
&& System.currentTimeMillis() < workload.getDuration().inMillisecs() + workload.getStarted()) {
log.println(workload + " interrupted before timer finished");
e.printStackTrace(); // something was wrong here
}
} finally {
// signal end
workload.finished();
log.println("Finished workload: " + workload);
workload.getHandler(ExecutionEvent.WorkloadCompletion).handle(workloadContext);
}
return 1L;
}
};
return scheduler;
}
private void execute(ITask task, Workload workload, ExecutionContext taskContext, PrintWriter log, int taskNumber, int round) throws InterruptedException {
try {
log.println(workload.getName() + "[" + taskNumber + "]->" + round);
task.run(taskContext);
} catch (Exception e) {
log.println("Task" + workload.getName() + "[" + taskNumber + "]" + " threw an exception while orderly execution: " + e.toString());
e.printStackTrace();
//throw new RuntimeException(e);
}
if (workload.hasDelay())
Thread.sleep(workload.getDelay());
}
}

2
src/main/java/net/engio/pips/lab/Range.java → src/main/java/net/engio/pips/lab/common/Range.java
View File

@ -1,4 +1,4 @@
package net.engio.pips.lab;
package net.engio.pips.lab.common;
import java.util.Arrays;
import java.util.Random;

2
src/main/java/net/engio/pips/lab/ValueGenerator.java → src/main/java/net/engio/pips/lab/common/ValueGenerator.java
View File

@ -1,4 +1,4 @@
package net.engio.pips.lab;
package net.engio.pips.lab.common;
/**
*

52
src/main/java/net/engio/pips/lab/workload/Workload.java
View File

@ -9,8 +9,8 @@ import java.util.Map;
import java.util.concurrent.TimeUnit;
/**
* A single workload defines a set of {@link ITask} to be executed as part of an {@link net.engio.pips.lab.Experiment}.
* Tasks are potentially run in parallel depending on the configuration of {@code setParallelTasks}.
* A single workload defines a set of {@link ITask} to be executed as part of an {@link net.engio.pips.lab.Benchmark}.
* Multiple tasks are run in parallel (see {@code setParallelTasks}).
* Tasks are created using the corresponding {@link net.engio.pips.lab.workload.ITaskFactory}.
*
*
@ -25,7 +25,7 @@ public class Workload {
private Duration duration;
private StartCondition starting = new StartCondition();
private StartCondition starting;
private String name;
@ -33,6 +33,8 @@ public class Workload {
private volatile long finished;
private long delay = -1;
private Map<ExecutionEvent, ExecutionHandlerWrapper> handlers = new HashMap<ExecutionEvent, ExecutionHandlerWrapper>();
@ -52,6 +54,15 @@ public class Workload {
return finished != -1;
}
public Workload setDelay(long ms){
delay = ms;
return this;
}
public boolean hasDelay(){
return delay > 0;
}
public long getExecutionTime(){
return isFinished() ? finished - started : -1;
}
@ -60,8 +71,19 @@ public class Workload {
return parallelUnits;
}
public boolean hasTasksToRun(){
return parallelUnits > 0;
}
/**
* Define how many task should run in parallel. Tasks are created using the
* specified {@link ITaskFactory}
*
* @param parallelUnits
* @return
*/
public Workload setParallelTasks(int parallelUnits) {
if(parallelUnits < 1 )throw new IllegalArgumentException("At least one task must run");
//if(parallelUnits < 1 )throw new IllegalArgumentException("At least one task must run");
this.parallelUnits = parallelUnits;
return this;
}
@ -74,12 +96,25 @@ public class Workload {
return ITaskFactory;
}
/**
* Set the task factory that will be used to create the single tasks of this workload.
*
* @param ITaskFactory The task factory to be used for task creation
* @return This workload
*/
public Workload setITaskFactory(ITaskFactory ITaskFactory) {
this.ITaskFactory = ITaskFactory;
return this;
}
/**
* Add an event handler to this workload. Depending on the type of event
* the handler will be called automatically by the {@link net.engio.pips.lab.Laboratory}
*
* @param event The type of event
* @param handler The handler to be invoked when the event occurs
* @return This workload
*/
public Workload handle(ExecutionEvent event, ExecutionHandler handler){
if(handlers.containsKey(event)){
handlers.get(event).delegate.add(handler);
@ -134,6 +169,11 @@ public class Workload {
return started;
}
public long getDelay() {
return delay;
}
// intermediate class for clean API
public class StartSpecification{
public Workload after(int timeout, TimeUnit unit){
@ -153,6 +193,7 @@ public class Workload {
}
// intermediate class for clean API
public class DurationSpecification{
public Workload lasts(int timeout, TimeUnit unit){
@ -172,6 +213,7 @@ public class Workload {
}
// wrap multiple execution handlers
public static class ExecutionHandlerWrapper implements ExecutionHandler{
private List<ExecutionHandler> delegate = new LinkedList<ExecutionHandler>();

10
src/main/java/net/engio/pips/reports/CSVFileExporter.java
View File

@ -1,7 +1,7 @@
package net.engio.pips.reports;
import net.engio.pips.data.IDataCollector;
import net.engio.pips.lab.Experiment;
import net.engio.pips.lab.Benchmark;
import java.io.File;
import java.io.PrintWriter;
@ -12,20 +12,20 @@ import java.io.PrintWriter;
*/
public class CSVFileExporter implements IReporter {
public void generate(Experiment experiment) throws Exception {
String reportDirectory = experiment.getReportBaseDir();
public void generate(Benchmark benchmark) throws Exception {
String reportDirectory = benchmark.getReportBaseDir();
File report = new File(reportDirectory + "report.txt");
PrintWriter writer = new PrintWriter(report);
try {
// write report header
writer.println("###### EXPERIMENT ##########");
writer.println(experiment);
writer.println(benchmark);
// write data of collectors
writer.println();
writer.println("##### COLLECTORS ########");
for (IDataCollector collector: experiment.getCollectors()) {
for (IDataCollector collector: benchmark.getCollectors()) {
writer.println(collector);
}

53
src/main/java/net/engio/pips/reports/ChartGenerator.java
View File

@ -1,20 +1,20 @@
package net.engio.pips.reports;
import net.engio.pips.lab.Experiment;
import net.engio.pips.lab.Benchmark;
import org.jfree.chart.ChartFactory;
import org.jfree.chart.ChartUtilities;
import org.jfree.chart.JFreeChart;
import org.jfree.chart.axis.AxisLocation;
import org.jfree.chart.axis.NumberAxis;
import org.jfree.chart.plot.XYPlot;
import org.jfree.chart.renderer.xy.XYLineAndShapeRenderer;
import org.jfree.data.time.TimeSeriesCollection;
import java.awt.*;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.*;
import java.util.List;
import java.util.Map;
/**
*
@ -50,29 +50,16 @@ public class ChartGenerator implements IReporter {
}
/**
* Configure a new group that will be treated as a single dataset in the chart.
* Each dataset has its own range axis.
* @param collectorId : The id used to retrieve associated data collectors from the result collector
* @param groupLabel : The label used for the range axis
* @return
*/
public SeriesGroup addGroup(String groupLabel, String collectorId){
SeriesGroup g = new SeriesGroup(groupLabel, collectorId);
groups.add(g);
return g;
}
public ChartGenerator add(SeriesGroup seriesGroup) {
public ChartGenerator draw(SeriesGroup seriesGroup) {
groups.add(seriesGroup);
return this;
}
public void generate(Experiment experiment){
public void generate(Benchmark benchmark){
Map<String, Integer> groupAxis = new HashMap<String, Integer>();
// process default group
SeriesGroup defaultGroup = this.groups.get(0);
TimeSeriesCollection collection = defaultGroup.createDataSet(experiment);
TimeSeriesCollection collection = defaultGroup.createDataSet(benchmark);
int maxNumberOfDatapoints = defaultGroup.getSize();
groupAxis.put(defaultGroup.getLabel(), 0);
@ -94,20 +81,25 @@ public class ChartGenerator implements IReporter {
plot.setDomainGridlinesVisible(true);
plot.setDomainGridlinePaint(Color.BLACK);
plot.setBackgroundPaint(Color.DARK_GRAY);
plot.setRenderer(0, getRandomRenderer());
// add other groups
List<SeriesGroup> groups = this.groups.subList(1, this.groups.size());
int axisIndex = 1,
dataSetIndex = 1;
for(SeriesGroup group : groups){
plot.setDataset(dataSetIndex, group.createDataSet(experiment));
plot.setDataset(dataSetIndex, group.createDataSet(benchmark));
plot.setRenderer(dataSetIndex, getRandomRenderer());
// if the group does not share a range axis with an already mapped group
if(!groupAxis.containsKey(group.getLabel())){
final NumberAxis axis2 = new NumberAxis(group.getLabel());
// prevent the axis to be scaled from zero if the dataset begins with higher values
axis2.setAutoRangeIncludesZero(false);
axis2.setAutoRangeIncludesZero(true);
plot.setRangeAxis(axisIndex, axis2);
plot.mapDatasetToRangeAxis(dataSetIndex, axisIndex);
plot.setRangeAxisLocation(axisIndex, group.getOrientation() == SeriesGroup.Orientation.Left
? AxisLocation.BOTTOM_OR_LEFT
: AxisLocation.BOTTOM_OR_RIGHT);
groupAxis.put(group.getLabel(), axisIndex); // remember this axis for subsequent groups
axisIndex++;
}
@ -124,12 +116,25 @@ public class ChartGenerator implements IReporter {
int width = maxNumberOfDatapoints * pixelPerDatapoint;
try {
String path = experiment.getReportBaseDir() + filename;
String path = benchmark.getReportBaseDir() + filename;
ChartUtilities.saveChartAsJPEG(new File(path), chart, width <= 1024 ? 1024 : width, 1024);
} catch (IOException e) {
System.err.println("Problem occurred creating chart.");
}
}
private Random rand = new Random();
private Color getRandomColor(){
return new Color(Math.abs(rand.nextInt()) % 256,// r
Math.abs(rand.nextInt()) % 256, // g
Math.abs(rand.nextInt()) % 256); //b
}
private XYLineAndShapeRenderer getRandomRenderer(){
XYLineAndShapeRenderer renderer = new XYLineAndShapeRenderer();
renderer.setSeriesPaint(0, getRandomColor());
return renderer;
}
}

4
src/main/java/net/engio/pips/reports/IReporter.java
View File

@ -1,6 +1,6 @@
package net.engio.pips.reports;
import net.engio.pips.lab.Experiment;
import net.engio.pips.lab.Benchmark;
/**
* Take the benchmark and create some output. This may be everything from logging
@ -11,5 +11,5 @@ import net.engio.pips.lab.Experiment;
*/
public interface IReporter {
void generate(Experiment experiment) throws Exception;
void generate(Benchmark benchmark) throws Exception;
}

74
src/main/java/net/engio/pips/reports/SeriesGroup.java
View File

@ -1,19 +1,14 @@
package net.engio.pips.reports;
import net.engio.pips.data.DataCollector;
import net.engio.pips.data.IDataCollector;
import net.engio.pips.data.IDataProcessor;
import net.engio.pips.data.aggregator.Average;
import net.engio.pips.data.utils.ItemCounter;
import net.engio.pips.data.utils.TimeBasedAggregator;
import net.engio.pips.lab.Experiment;
import net.engio.pips.lab.Benchmark;
import org.jfree.data.time.TimeSeries;
import org.jfree.data.time.TimeSeriesCollection;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.List;
/**
* A series group is used to configure a set of data collectors for being included in a time series
@ -25,30 +20,44 @@ import java.util.Map;
*/
public class SeriesGroup {
private String collectorId;
public static enum Orientation{
Left, Right
}
private String label;
private Map<String, IDataProcessor> aggregators = new HashMap<String, IDataProcessor>();
private Collection<IDataCollector> collectors = new ArrayList<IDataCollector>();
private int size;
private String yAxis = "";
private int collectorSampleSize = 1;
private Orientation orientation = Orientation.Left;
public SeriesGroup(String collectorId, String label) {
this.collectorId = collectorId;
public SeriesGroup(String label) {
this.label = label;
}
public SeriesGroup setYAxisOrientation(Orientation orientation){
this.orientation = orientation;
return this;
}
public Orientation getOrientation() {
return orientation;
}
public SeriesGroup addCollector(IDataCollector collector){
collectors.add(collector);
return this;
}
public SeriesGroup addCollectors(List<IDataCollector> collectors){
for(IDataCollector collector : collectors)
addCollector(collector);
return this;
}
public String getLabel() {
return label;
}
@ -57,15 +66,6 @@ public class SeriesGroup {
return size;
}
public String getCollectorId() {
return collectorId;
}
public SeriesGroup aggregate(String name, IDataProcessor aggregator){
aggregators.put(name, aggregator);
return this;
}
public String getyAxis() {
return yAxis;
}
@ -75,45 +75,21 @@ public class SeriesGroup {
return this;
}
public SeriesGroup setDrawEveryNthGraph(int factor) {
this.collectorSampleSize = factor;
return this;
}
public TimeSeriesCollection createDataSet(Experiment experiment){
Collection<IDataCollector> collectors = experiment.getCollectors(collectorId);
collectors.addAll(this.collectors);
public TimeSeriesCollection createDataSet(Benchmark benchmark){
TimeSeriesCollection collection = new TimeSeriesCollection();
TimeBasedAggregator aggregator = new TimeBasedAggregator();
// create a series from each data collector
int numberOfCollectors = 1;
for(IDataCollector collector : collectors){
// ignore empty data collectors as well as according to sample size
if(collector == null || collector.size() == 0)continue;
if(numberOfCollectors % collectorSampleSize != 0){
numberOfCollectors++;
continue;
}
TimeSeriesConsumer wrapper = new TimeSeriesConsumer(collector.getId());
collector.feed(wrapper);
TimeSeries series = wrapper.getSeries();
collection.addSeries(series);
// prepare the time based aggregator
if(!aggregators.isEmpty())
aggregator.consume(collector);
numberOfCollectors++;
if(size < collector.size())size = collector.size();
}
DataCollector average = aggregator.fold(new Average());
ItemCounter numberOfDatapoints = new ItemCounter();
for(Map.Entry<String, IDataProcessor> aggregation : aggregators.entrySet()){
TimeSeriesConsumer series = new TimeSeriesConsumer(aggregation.getKey());
IDataProcessor aggregate = aggregation.getValue();
aggregate.pipeInto(numberOfDatapoints).pipeInto(series);
average.feed(aggregate);
collection.addSeries(series.getSeries());
}
size = numberOfDatapoints.getItemCount();
return collection;
}
}

10
src/test/java/net/engio/lab/ExecutionContextTest.java
View File

@ -1,8 +1,8 @@
package net.engio.lab;
import net.engio.pips.lab.Benchmark;
import net.engio.pips.lab.ExecutionContext;
import net.engio.pips.lab.Executions;
import net.engio.pips.lab.Experiment;
import org.junit.Test;
import java.util.Collection;
@ -20,12 +20,12 @@ public class ExecutionContextTest extends UnitTest{
Map<String, Object> bindings = new HashMap<String, Object>();
bindings.put(ExecutionContext.class.toString(), ExecutionContext.class);
bindings.put(UnitTest.class.toString(), UnitTest.class);
bindings.put(Experiment.class.toString(), Experiment.class);
bindings.put(Benchmark.class.toString(), Benchmark.class);
return bindings;
}
private ExecutionContext getInitialContext(Map<String, Object> bindings){
ExecutionContext ctx = new ExecutionContext(new Experiment("test"));
ExecutionContext ctx = new ExecutionContext(new Benchmark("test"));
ctx.bindAll(bindings);
assertBindingsExist(bindings, ctx);
@ -104,7 +104,7 @@ public class ExecutionContextTest extends UnitTest{
@Test
public void testGetMatching(){
ExecutionContext ctx = new ExecutionContext(new Experiment("test"));
ExecutionContext ctx = new ExecutionContext(new Benchmark("test"));
Object[] bindings = new Object[]{"root", "root:lvl1", "root:lvl1:lvl2", "none"};
ctx.bind(bindings);
@ -130,7 +130,7 @@ public class ExecutionContextTest extends UnitTest{
@Test
public void testExecutions(){
ExecutionContext ctx = new ExecutionContext(new Experiment("test"));
ExecutionContext ctx = new ExecutionContext(new Benchmark("test"));
Object[] bindings = new Object[]{"root", "root:lvl1", "root:lvl1:lvl2", "none"};
ctx.bind(bindings);

157
src/test/java/net/engio/lab/LaboratoryTest.java
View File

@ -1,7 +1,8 @@
package net.engio.lab;
import net.engio.pips.lab.Benchmark;
import net.engio.pips.lab.ExecutionContext;
import net.engio.pips.lab.Experiment;
import net.engio.pips.lab.LabException;
import net.engio.pips.lab.Laboratory;
import net.engio.pips.lab.workload.*;
import org.junit.Test;
@ -29,8 +30,81 @@ public class LaboratoryTest extends UnitTest{
}
};
@Test
public void testInvalidWorkloadSetups() throws Exception {
Workload withoutFactory = new Workload("without factory")
.starts().immediately()
.duration().repetitions(1);
Workload withoutStart = new Workload("without start")
.setITaskFactory(NoOperation)
.duration().repetitions(1);
Workload withoutDuration = new Workload("without duration")
.starts().immediately()
.setITaskFactory(NoOperation);
Workload cyclicStart1 = new Workload("without duration");
Workload cyclicStart2 = new Workload("without duration");
cyclicStart1
.starts().after(cyclicStart2)
.duration().repetitions(1)
.setITaskFactory(NoOperation);
cyclicStart2
.starts().after(cyclicStart1)
.duration().repetitions(1)
.setITaskFactory(NoOperation);
Workload cyclicDuration1 = new Workload("without duration");
Workload cyclicDuration2 = new Workload("without duration");
cyclicDuration1
.starts().immediately()
.duration().depends(cyclicDuration2)
.setITaskFactory(NoOperation);
cyclicDuration2
.starts().immediately()
.duration().depends(cyclicDuration1)
.setITaskFactory(NoOperation);
Laboratory lab = new Laboratory();
try {
lab.run(new Benchmark("Invalid").addWorkload(withoutFactory));
fail();
} catch (LabException e) {
assertEquals(e.getCode(), LabException.ErrorCode.WLWithoutFactory);
}
try {
lab.run(new Benchmark("Invalid").addWorkload(withoutStart));
fail();
} catch (LabException e) {
assertEquals(e.getCode(), LabException.ErrorCode.WLWithoutStart);
}
try {
lab.run(new Benchmark("Invalid").addWorkload(withoutDuration));
fail();
} catch (LabException e) {
assertEquals(e.getCode(), LabException.ErrorCode.WLWithoutDuration);
}
try {
lab.run(new Benchmark("Invalid").addWorkload(cyclicStart1, cyclicStart2));
fail();
} catch (LabException e) {
assertEquals(e.getCode(), LabException.ErrorCode.WLWithCycleInStart);
}
try {
lab.run(new Benchmark("Invalid").addWorkload(cyclicDuration1, cyclicDuration2));
fail();
} catch (LabException e) {
assertEquals(e.getCode(), LabException.ErrorCode.WLWithCycleInDuration);
}
}
@Test
public void testWorkloadSchedulingStartingAfter() throws Exception {
@ -75,17 +149,18 @@ public class LaboratoryTest extends UnitTest{
.starts().after(first);
Laboratory lab = new Laboratory();
lab.run(new Experiment("test").addWorkload(first, second));
lab.run(new Benchmark("test").addWorkload(first, second));
Thread.sleep(100);
// both have run in the end
assertEquals(0, counter.get());
}
@Test
public void testWorkloadShutdownCancelsTasks() throws Exception {
final AtomicInteger counter = new AtomicInteger(1);
Workload countUp = new Workload("First workload")
//@Test
// TODO: this invalid start/end dependency tree must be fixed
public void testWorkloadSchedulingStartingAfterDelayed() throws Exception {
final AtomicInteger counter = new AtomicInteger(0);
Workload first = new Workload("First workload")
.setParallelTasks(15)
.setITaskFactory(new ITaskFactory() {
@Override
@ -98,6 +173,58 @@ public class LaboratoryTest extends UnitTest{
};
}
})
.duration().repetitions(1)
.starts().after(2, TimeUnit.SECONDS);
Workload second = new Workload("Second Workload")
.setParallelTasks(15)
.setITaskFactory(new ITaskFactory() {
@Override
public ITask create(ExecutionContext context) {
return new ITask() {
@Override
public void run(ExecutionContext context) throws Exception {
counter.decrementAndGet();
}
};
}
})
.handle(ExecutionEvent.WorkloadInitialization, new ExecutionHandler() {
@Override
public void handle(ExecutionContext context) {
// the first workload
assertEquals(15, counter.get());
}
})
.duration().depends(first)
.starts().after(first);
Laboratory lab = new Laboratory();
lab.run(new Benchmark("test").addWorkload(first, second));
Thread.sleep(4000);
// both have run in the end
assertEquals(0, counter.get());
}
@Test
public void testWorkloadShutdownCancelsTasks() throws Exception {
final AtomicInteger first = new AtomicInteger(1);
final AtomicInteger second = new AtomicInteger(1);
Workload countUp = new Workload("First workload")
.setParallelTasks(15)
.setITaskFactory(new ITaskFactory() {
@Override
public ITask create(ExecutionContext context) {
return new ITask() {
@Override
public void run(ExecutionContext context) throws Exception {
first.incrementAndGet();
}
};
}
})
.duration().lasts(5, TimeUnit.SECONDS)
.starts().immediately();
@ -109,7 +236,7 @@ public class LaboratoryTest extends UnitTest{
return new ITask() {
@Override
public void run(ExecutionContext context) throws Exception {
counter.decrementAndGet();
second.incrementAndGet();
}
};
}
@ -118,14 +245,16 @@ public class LaboratoryTest extends UnitTest{
.starts().immediately();
Laboratory lab = new Laboratory();
lab.run(new Experiment("test").addWorkload(countUp, countDown));
lab.run(new Benchmark("test").addWorkload(countUp, countDown));
int count = counter.get();
int firstVal = first.get();
int secondVal = second.get();
// no threads that change the counter are running anymore
Thread.sleep(1000);
assertEquals(count, counter.get());
// both have run in the end
assertTrue(counter.get() < 16);
assertEquals(firstVal, first.get());
assertEquals(secondVal, second.get());
assertTrue(firstVal > 1);
assertTrue(secondVal > 1);
}
@Test
@ -166,7 +295,7 @@ public class LaboratoryTest extends UnitTest{
.starts().immediately();
Laboratory lab = new Laboratory();
lab.run(new Experiment("test").addWorkload(first, second));
lab.run(new Benchmark("test").addWorkload(first, second));
// first workload has run at least duration seconds
assertTrue(finish.get() - start.get() >= duration * 1000);
@ -203,7 +332,7 @@ public class LaboratoryTest extends UnitTest{
});
Laboratory lab = new Laboratory();
lab.run(new Experiment("test").addWorkload(first, second, third));
lab.run(new Benchmark("test").addWorkload(first, second, third));
assertTrue(finished.get());
}