Skip to content
Snippets Groups Projects
Commit 9736d76b authored by fjn1g13's avatar fjn1g13
Browse files

Arbitrary max parallelism

parent f8e1dfc9
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
IVMCTrim/Program.cs
+ 67
− 31
View file @ 9736d76b
......@@ -23,7 +23,8 @@ namespace IVMCTrim
MathNet.Numerics.Control.UseSingleThread();
// repeats with Z=0.95
RunRepeats("Z095", Configurations.Standard(Z: 0.95), baseSeed: 2042, repeats: 4);
RunRepeats("Z095", Configurations.Standard(Z: 0.95), baseSeed: 2042, repeats: 40, startFrom: 4);
return;
// runs a bunch of experiments with the 'standard' configuration, only varying Z
// seeds should match those used in the paper
......@@ -55,24 +56,24 @@ namespace IVMCTrim
var ctx = new ExecutionContext(new MersenneTwister(seed, false));
// run the actual experiment
Model.RunExperiment(ctx, parameters, reporter);
var runonFrequency = Model.RunExperiment(ctx, parameters, reporter);
// produce output from recorders
Recording.ProduceOutput(outputDirectory, parameters, recorders);
// TODO: should probably do run-on solve count
Console.WriteLine($"{outputDirectory} runon frequency: {runonFrequency}");
}
/// <summary>
/// Runs many experiments in parallel with the same parameters and sequential seeds.
/// </summary>
public static void RunRepeats(string outputDirectory, Parameters parameters, int baseSeed, int repeats, int reportingPeriodMultiplier = 1)
public static void RunRepeats(string outputDirectory, Parameters parameters, int baseSeed, int repeats, int startFrom = 0, int reportingPeriodMultiplier = 1)
{
var tasks = new List<Task>();
var tasks = new List<Action>();
int runonSolveCount = 0;
baseSeed += startFrom;
for (int r = 0; r < repeats; r++)
for (int r = startFrom; r < repeats + startFrom; r++)
{
// incremen seeds
int seed = baseSeed++;
......@@ -81,12 +82,12 @@ namespace IVMCTrim
var dir = System.IO.Path.Combine(outputDirectory, $"r{r}");
// run
tasks.Add(Task.Run(() => Run(dir, parameters, seed, reportingPeriodMultiplier)));
tasks.Add(new Action(() => Run(dir, parameters, seed, reportingPeriodMultiplier)));
}
// wait until they've all finished
// (scheduling will be down to the thread pool, so it may be unpredictable)
Task.WaitAll(tasks.ToArray());
Parallel.ForEach(tasks, new ParallelOptions { MaxDegreeOfParallelism = 8 }, x => x());
}
}
......@@ -112,7 +113,8 @@ namespace IVMCTrim
/// <summary>
/// Runs a single experiment with the given parameters in the given context, reporting the completion of each episode.
/// </summary>
public static void RunExperiment(ExecutionContext ctx, Parameters parameters, EndOfEpisodeReporter endOfEpisode)
/// <returns>The runon frequency.</returns>
public static double RunExperiment(ExecutionContext ctx, Parameters parameters, EndOfEpisodeReporter endOfEpisode)
{
// prepare the initial individual
int N = parameters.ModuleCount * parameters.ModuleSize;
......@@ -134,6 +136,10 @@ namespace IVMCTrim
// report state at end of episode
endOfEpisode?.Invoke(environment, result, episode);
}
// compute runon stuff
var runonFrequency = MeasureRunonSolveFrequency(ctx, parameters, individual);
return runonFrequency;
}
/// <summary>
......@@ -198,6 +204,29 @@ namespace IVMCTrim
return new Evaluation(fitness, benefit, cost);
}
/// <summary>
/// Performs a run-on, and counts the proportion of solves.
/// </summary>
/// <remarks>
/// So long as the variation in the environment has a unbiased distribution, then this is an unbiased runon.
/// </remarks>
public static double MeasureRunonSolveFrequency(ExecutionContext ctx, Parameters parameters, Individual individual, int runonEpisodeCount = 100)
{
individual = individual.Clone();
int[] wins = new int[parameters.ModuleCount + 1];
var environment = new IvmcEnvironment(parameters.ModuleSize, parameters.ModuleCount);
for (int episode = 0; episode < runonEpisodeCount; episode++)
{
var result = Model.RunEpisode(episode, ctx, parameters, environment, ref individual, null);
Recorders.CountSolves(environment, result.Inidividual, result.Evaluation, wins);
}
return (double)wins.Last() / runonEpisodeCount;
}
}
public static class Configurations
......@@ -296,7 +325,7 @@ namespace IVMCTrim
public class Parameters
{
// default should be equivalent to the 'main' figure in the poaper
// m4m gen=composedense bprob=0.5 BEx=true lambda=0.1 Z=0.95 regfunc=l1 ch=1.0 cl=0.7 c0=-1.0 epochs=120000 mb=1E-3 judgemode=Split K=1000 environmentpackage=ivmc:4 transmatmutator=singlecell
// m4m gen=composedense bprob=0.5 BEx=true lambda=0.1 Z=0.95 regfunc=l1 ch=1.0 cl=0.7 c0=-1.0 epochs=400000 mb=1E-3 judgemode=Split K=1000 environmentpackage=ivmc:4 transmatmutator=singlecell
public int ModuleCount { get ; set; } = 4;
......@@ -340,7 +369,7 @@ namespace IVMCTrim
public double MutateBProbability { get; set; } = 0.5;
public double RB => MutateBProbability;
public int TotalEpisodes { get; set; } = 120_000;
public int TotalEpisodes { get; set; } = 400_000;
public EpisodeStartOperation EpisodeStartOperation { get; set; } = EpisodeStartOperations.VaryEnvironment;
}
......@@ -847,8 +876,13 @@ namespace IVMCTrim
if (episode % (reportingPeriodMultiplier * 1000) == 0)
{
// this is totally pointless: we have the win count already, which is just as informative
// runon solve frequency
var ctx = new ExecutionContext(new MersenneTwister()); // use random seed: we don't need these to be reproducable, and we don't want any sampling bias
var runonFrequency = 0;// Model.MeasureRunonSolveFrequency(ctx, parameters, individual);
// report progress
Console.WriteLine($"{outputDirectory}: {episode}/{parameters.TotalEpisodes} ({(double)episode / parameters.TotalEpisodes:P})\tf={evaluation.Fitness}");
Console.WriteLine($"{outputDirectory}: {episode}/{parameters.TotalEpisodes} ({(double)episode / parameters.TotalEpisodes:P})\tf={evaluation.Fitness}\t{runonFrequency:P}");
// produce grn plot
var grnPlot = PlotHelpers.IntegralCartesianAxes($"Episode {episode}", "row", "col");
......@@ -857,7 +891,7 @@ namespace IVMCTrim
grnPlot.ExportPdf(path($"episode{episode}.pdf"), 0.5, true);
// run tracee
var ctx = new ExecutionContext(new MersenneTwister(3)); // common seed (chosen so that it has a mix of cL and C0)
ctx = new ExecutionContext(new MersenneTwister(3)); // common seed (chosen so that it has a mix of cL and C0)
var env = new IvmcEnvironment(parameters.ModuleSize, parameters.ModuleCount);
var clone = individual.Clone(); // don't want to modify the real guy
......@@ -894,54 +928,56 @@ namespace IVMCTrim
{
string path(string f) => System.IO.Path.Combine(outDir, f);
const double plotSize = 0.5;
// data export and mindless plotting
var rcsData = recorders.GetRcsDataFlat();
TrajectoryHelpers.SaveTrajectories(path("rcs_t.dat"), rcsData, recorders.Grns.SamplePeriod);
var rcsPlot = PlotHelpers.LinearAxes("Regulation Coefficients", "Episode", "Connection Strength");
var rcsPlot = PlotHelpers.LinearAxes("Regulation Coefficients", "Episodes", "Connection Strength");
rcsPlot.Series.AddRange(PlotHelpers.PlotTrajectoryLines2D(recorders.GetRcsData(), recorders.Grns.SamplePeriod, OxyColors.DarkCyan, OxyColors.LightCyan, OxyColors.DarkGreen, OxyColors.LightGreen));
rcsPlot.ExportPdf(path("rcs_t.pdf"), 0.5, false);
rcsPlot.ExportPdf(path("rcs_t.pdf"), plotSize, false);
var hierarchyPlot = PlotHelpers.LinearAxes("Evolution of Hierarchy", "Episode", "Degree of Hierarchy");
var hierarchyPlot = PlotHelpers.LinearAxes("Evolution of Hierarchy", "Episodes", "Degree of Hierarchy");
hierarchyPlot.Series.AddRange(
PlotHelpers.PlotTrajectoryLines1D(
recorders.Grns.Samples.Select(grn => Analysis.ComputeBlockModuleHierarchy(grn, parameters.ModuleSize).ToList()).ToArray().Transpose(),
recorders.Grns.SamplePeriod,
OxyColors.DarkSlateGray, OxyColors.SlateGray));
hierarchyPlot.ExportPdf(path("hierarchy_t.pdf"), 0.5, false);
hierarchyPlot.ExportPdf(path("hierarchy_t.pdf"), plotSize, false);
var moduleIndependencePlot = PlotHelpers.LinearAxes("Evolution of ModuleIndependence", "Episode", "Degree of Hierarchy");
var moduleIndependencePlot = PlotHelpers.LinearAxes("Evolution of ModuleIndependence", "Episodes", "Degree of Hierarchy");
moduleIndependencePlot.Series.AddRange(
PlotHelpers.PlotTrajectoryLines1D(
recorders.Grns.Samples.Select(grn => Analysis.ComputeBlockModuleIndependence(grn, parameters.ModuleSize).ToList()).ToArray().Transpose(),
recorders.Grns.SamplePeriod,
OxyColors.Teal, OxyColors.DarkTurquoise));
moduleIndependencePlot.ExportPdf(path("moduleIndependence_t.pdf"), 0.5, false);
moduleIndependencePlot.ExportPdf(path("moduleIndependence_t.pdf"), plotSize, false);
var fitnessData = recorders.GetFitnessData();
TrajectoryHelpers.SaveTrajectories(path("fitness_t.dat"), fitnessData, recorders.Evaluations.SamplePeriod);
var fitnessPlot = PlotHelpers.LinearAxes("Fitness", "Episode", "Fitness");
var fitnessPlot = PlotHelpers.LinearAxes("Fitness", "Episodes", "Fitness");
fitnessPlot.Series.AddRange(PlotHelpers.PlotTrajectoryScatters1D(fitnessData.Take(1).ToArray().DownsampleTrajectories(100), recorders.Evaluations.SamplePeriod * 100, OxyColors.Red, OxyColors.DarkOrange));
fitnessPlot.ExportPdf(path("fitness_t.pdf"), 0.5, false);
fitnessPlot.ExportPdf(path("fitness_t.pdf"), plotSize, false);
var meanFitnessPlot = PlotHelpers.LinearAxes("Mean Fitness", "Episode", "Mean Fitness");
fitnessPlot.Series.AddRange(PlotHelpers.PlotTrajectoryScatters1D(fitnessData.Take(1).ToArray().MeanDownsampleTrajectories(100), recorders.Evaluations.SamplePeriod * 100, OxyColors.Red, OxyColors.DarkOrange));
fitnessPlot.ExportPdf(path("meanfitness_t.pdf"), 0.5, false);
var meanFitnessPlot = PlotHelpers.LinearAxes("Mean Fitness", "Episodes", "Mean Fitness");
meanFitnessPlot.Series.AddRange(PlotHelpers.PlotTrajectoryScatters1D(fitnessData.Take(1).ToArray().MeanDownsampleTrajectories(100), recorders.Evaluations.SamplePeriod * 100, OxyColors.Red, OxyColors.DarkOrange));
meanFitnessPlot.ExportPdf(path("meanfitness_t.pdf"), plotSize, false);
var solveData = recorders.GetSolveData().Map2D(x => (double)x);
TrajectoryHelpers.SaveTrajectories(path("ivmcsolves_t.dat"), solveData, recorders.Solves.SamplePeriod);
var solvePlot = PlotHelpers.LinearAxes("Solves", "Episode", "Solve Frequency");
var solvePlot = PlotHelpers.LinearAxes("Solves", "Episodes", "Solve Frequency");
solvePlot.Series.AddRange(PlotHelpers.PlotTrajectoryScatters1D(solveData.Map2D(x => x / recorders.Solves.SamplePeriod).TakeLast(1).ToArray(), recorders.Solves.SamplePeriod, OxyColors.IndianRed, OxyColors.DarkRed));
solvePlot.ExportPdf(path("solves_t.pdf"), 0.5, false);
solvePlot.ExportPdf(path("solves_t.pdf"), plotSize, false);
var switchData = recorders.GetSwitchData().Map2D(x => (double)x);
TrajectoryHelpers.SaveTrajectories(path("ivmcswitches_t.dat"), switchData, recorders.Switches.SamplePeriod);
var switchPlot = PlotHelpers.LinearAxes("Switches", "Episode", "Switch Frequency");
var switchPlot = PlotHelpers.LinearAxes("Switches", "Episodes", "Switch Frequency");
switchPlot.Series.AddRange(PlotHelpers.PlotTrajectoryScatters1D(switchData.Map2D(x => x / recorders.Switches.SamplePeriod).TakeLast(1).ToArray(), recorders.Switches.SamplePeriod, OxyColors.DeepPink, OxyColors.HotPink));
switchPlot.ExportPdf(path("switches_t.pdf"), 0.5, false);
switchPlot.ExportPdf(path("switches_t.pdf"), plotSize, false);
}
}
......@@ -1021,7 +1057,7 @@ namespace IVMCTrim
Switches = new CountRecorder(moduleCount + 1, CountSwitches(), OtherSamplePeriod * samplePeriodMultiplier);
}
private static Action<IvmcEnvironment, Individual, Evaluation, int[]> CountSwitches()
public static Action<IvmcEnvironment, Individual, Evaluation, int[]> CountSwitches()
{
Vector<double> last = null;
......@@ -1065,7 +1101,7 @@ namespace IVMCTrim
return count;
}
private static void CountSolves(IvmcEnvironment environment, Individual individual, Evaluation Evaluation, int[] wins)
public static void CountSolves(IvmcEnvironment environment, Individual individual, Evaluation Evaluation, int[] wins)
{
if (environment == null)
return; // pre-start
......
readme.md
+ 3
− 3
View file @ 9736d76b
# IVMCTrim
This is a (somewhat) minimal implementation of a HillClimber/IVMC experiment from M4M. It runs exactly the same experiments (given the same parameters), but the trajectory are not bit-for-bit idential (due to some weirdness in M4M to do with supporting population models).
This is a (somewhat) minimal implementation of a HillClimber/IVMC experiment from M4M. It runs exactly the same experiments (given the same parameters), producing the same output (which requires a couple of weird implementations details, but nothing too painful).
## Dependencies
......@@ -32,7 +32,7 @@ This will run each of the experiments called in the `Program.Main` method. Each
## Notes
The idea behind IVMCTrim is that it is possible to understand/mess with the model quickly, instead of having to trawl through M4M looking for the right place to plug in an `IComposer` or whatever, only to find that your code doesn't serialise properly. If you want to understand the model: this is the place to look. If you want to perform serious analysis, then you will want M4M.
The idea behind IVMCTrim is that it is possible to understand/mess with the model quickly, instead of having to trawl through M4M looking for the right place to plug in an `IComposer` or whatever, only to find that your code doesn't serialise properly. If you want to understand the model or just want to reproduce existing results, IVMCTrim is the place to look. If you want to perform serious analysis, then you will want M4M.
In order to ensure the simulations are exactly the same as those run under M4M, the order and method by which random numbers are generated must be exactly the same. This ordering is fairly easy to see in IVMCTrim.
......@@ -40,7 +40,7 @@ In order to ensure the simulations are exactly the same as those run under M4M,
## Help / More / Contact
If you run into trouble or want access to M4M (the nightmare inducing program that ran the actual experiments in the paper) then contact fjn1g13@soton.ac.uk. It is not made available 'by default' due to the complexity of the implementation, and the many unfinished and broken components it contains: we believe it is better to supply an understandable implementation, but if you want to reproduce the figures from the paper or otherwise mess with the model/analysis, then I'll be happy to help.
If you run into trouble or want access to M4M (the nightmare inducing program that ran the original experiments in the paper) then contact fjn1g13@soton.ac.uk. It is not made available 'by default' due to the complexity of the implementation, the many unfinished and broken components it contains, and the complexity involved in using it: we believe it is better to supply an understandable implementation, but if you want to reproduce the figures from the paper exactly or otherwise mess with the model/analysis, then I'll be happy to help.
## FAQ
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment