Hello, first new thread here.  I would like to be able to call the C#.NET QC engine multiple times without exceptions, and without contamination the results between algorithm runs.  Is this possible with the current QC Lean Git project?  If so, can someone show how that can be done in the modified Launcher Program.cs code below?  In the code block below you can see the gist of what I am trying to do; run the engine multiple times with different parameters to run multiple algorithms.  The next step would be to get separate result objects for each algorithm run.  The below code will run and throw some exceptions that are caught with the engine then returning identical result objects per run; with the second result object returned identical to the first since the second run failed.

So far I have been looking through to code to try to find some means of 'resetting' the engine or job state between runs, with the hope that there is some already implemented means of accomplishing this.  Right now I am getting some exception in WorkerThread.cs Add method, "System.InvalidOperationException: 'The collection has been marked as complete with regards to additions.'"  Which seems due to the WorkerThread class Dispose method setting CompleteAdding on the _blockingCollection variable.  One thing that I have tried is instantiate separate engine, algorithm handler, and job objects but still the WorkerThread is not reset and throws the same exception.

I figure it would be easier to ask this here rather than go down the rabbit hole of hacking at the QC project code if this solution is already among the code.  I found some posts, but they were either old or did not definitively conclude if this can be done within QC Lean.  Please clue me in.

/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ using System; using System.ComponentModel.Composition; using System.IO; using System.Threading; using QuantConnect.Configuration; using QuantConnect.Lean.Engine; using QuantConnect.Logging; using QuantConnect.Packets; using QuantConnect.Util; namespace QuantConnect.Lean.Launcher { public class Program { private const string _collapseMessage = "Unhandled exception breaking past controls and causing collapse of algorithm node. This is likely a memory leak of an external dependency or the underlying OS terminating the LEAN engine."; private static LeanEngineSystemHandlers leanEngineSystemHandlers; private static LeanEngineAlgorithmHandlers leanEngineAlgorithmHandlers; private static AlgorithmNodePacket job; private static AlgorithmManager algorithmManager; static Program() { AppDomain.CurrentDomain.AssemblyLoad += (sender, e) => { if (e.LoadedAssembly.FullName.ToLowerInvariant().Contains("python")) { Log.Trace($"Python for .NET Assembly: {e.LoadedAssembly.GetName()}"); } }; } static void Main(string[] args) { Run(new string[] { "--algorithm-type-name MACDTrendAlgorithm" }); Run(new string[] { "--algorithm-type-name MovingAverageCrossAlgorithm" }); } private static void Run(string[] args) { //Initialize: var mode = "RELEASE"; #if DEBUG mode = "DEBUG"; #endif if (OS.IsWindows) { Console.OutputEncoding = System.Text.Encoding.UTF8; } // expect first argument to be config file name if (args.Length > 0) { Config.MergeCommandLineArgumentsWithConfiguration(LeanArgumentParser.ParseArguments(args)); } var environment = Config.Get("environment"); var liveMode = Config.GetBool("live-mode"); Log.DebuggingEnabled = Config.GetBool("debug-mode"); Log.FilePath = Path.Combine(Config.Get("results-destination-folder"), "log.txt"); Log.LogHandler = Composer.Instance.GetExportedValueByTypeName<ILogHandler>(Config.Get("log-handler", "CompositeLogHandler")); //Name thread for the profiler: if (Thread.CurrentThread.Name == null) { Thread.CurrentThread.Name = "Algorithm Analysis Thread"; } Log.Trace("Engine.Main(): LEAN ALGORITHMIC TRADING ENGINE v" + Globals.Version + " Mode: " + mode + " (" + (Environment.Is64BitProcess ? "64" : "32") + "bit)"); Log.Trace("Engine.Main(): Started " + DateTime.Now.ToShortTimeString()); //Import external libraries specific to physical server location (cloud/local) try { leanEngineSystemHandlers = LeanEngineSystemHandlers.FromConfiguration(Composer.Instance); } catch (CompositionException compositionException) { Log.Error("Engine.Main(): Failed to load library: " + compositionException); throw; } //Setup packeting, queue and controls system: These don't do much locally. leanEngineSystemHandlers.Initialize(); //-> Pull job from QuantConnect job queue, or, pull local build: string assemblyPath; job = leanEngineSystemHandlers.JobQueue.NextJob(out assemblyPath); if (job == null) { const string jobNullMessage = "Engine.Main(): Sorry we could not process this algorithm request."; Log.Error(jobNullMessage); throw new ArgumentException(jobNullMessage); } try { leanEngineAlgorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(Composer.Instance); } catch (CompositionException compositionException) { Log.Error("Engine.Main(): Failed to load library: " + compositionException); throw; } // if the job version doesn't match this instance version then we can't process it // we also don't want to reprocess redelivered jobs if (job.Redelivered) { Log.Error("Engine.Run(): Job Version: " + job.Version + " Deployed Version: " + Globals.Version + " Redelivered: " + job.Redelivered); //Tiny chance there was an uncontrolled collapse of a server, resulting in an old user task circulating. //In this event kill the old algorithm and leave a message so the user can later review. leanEngineSystemHandlers.Api.SetAlgorithmStatus(job.AlgorithmId, AlgorithmStatus.RuntimeError, _collapseMessage); leanEngineSystemHandlers.Notify.SetAuthentication(job); leanEngineSystemHandlers.Notify.Send(new RuntimeErrorPacket(job.UserId, job.AlgorithmId, _collapseMessage)); leanEngineSystemHandlers.JobQueue.AcknowledgeJob(job); Exit(1); } try { // Set our exit handler for the algorithm Console.CancelKeyPress += new ConsoleCancelEventHandler(ExitKeyPress); // Create the algorithm manager and start our engine algorithmManager = new AlgorithmManager(liveMode, job); leanEngineSystemHandlers.LeanManager.Initialize(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, job, algorithmManager); var engine = new Engine.Engine(leanEngineSystemHandlers, leanEngineAlgorithmHandlers, liveMode); engine.Run(job, algorithmManager, assemblyPath, WorkerThread.Instance); } finally { var algorithmStatus = algorithmManager?.State ?? AlgorithmStatus.DeployError; Exit(algorithmStatus != AlgorithmStatus.Completed ? 1 : 0); } } public static void ExitKeyPress(object sender, ConsoleCancelEventArgs args) { // Allow our process to resume after this event args.Cancel = true; // Stop the algorithm algorithmManager.SetStatus(AlgorithmStatus.Stopped); Log.Trace("Program.ExitKeyPress(): Lean instance has been cancelled, shutting down safely now"); } public static void Exit(int exitCode) { //Delete the message from the job queue: leanEngineSystemHandlers.JobQueue.AcknowledgeJob(job); Log.Trace("Engine.Main(): Packet removed from queue: " + job.AlgorithmId); // clean up resources leanEngineSystemHandlers.DisposeSafely(); leanEngineAlgorithmHandlers.DisposeSafely(); Log.LogHandler.DisposeSafely(); OS.CpuPerformanceCounter.DisposeSafely(); Log.Trace("Program.Main(): Exiting Lean..."); //Environment.Exit(exitCode); } } }