namespace QuantConnect.Algorithm.CSharp
{   
	/*
	Illustrates how to access Amazon S3 from QuantConnect cloud.
	
	Because the whole point of this S3 configuration is to prevent unauthorized access,
	I can't share a bucket without additional effort. Configuring Amazon S3 buckets is
	unfortunately a bit of work (at least if you do it rarely like I do or for the first time).
	
	Examples of what you can do this code with some modification:
	1) Warm up algorithm through backtests, with restored state from S3 when launching live (this example) - S3State
	2) Update parameters or signals by polling S3 bucket (not efficient and pretty ugly, but it works) - S3ParamSet
	3) Some kind of external logging - S3Client generic usage
	
	This comes without any warranty of any kind. Use at your own risk.
	*/
    public class S3ExampleAlgorithm : QCAlgorithm
    {
    	private const string _algoID = "helloworld";
    
        public override void Initialize() 
        {
            SetStartDate(2016, 1, 1);         
            SetEndDate(DateTime.Now);
            SetCash(25000);
            
            //at a minimum, you need to replace xxx, yyy, and zzz below
            var s3Client = new S3Client();
            s3Client.BaseURL = new Uri("https://s3-eu-west-1.amazonaws.com");
            s3Client.UserID = "xxx";
            s3Client.Password = "yyy";
            _s3state = new S3State(this, s3Client);
            _s3state.Bucket = "zzz";
            _s3state.AlgoID = _algoID;
            
            _assets.Add(new Asset(this, "SPY"));
            
            _s3state.DownloadWarmupState(_assets);
        }
        
        private S3State _s3state;
    	private readonly List<Asset> _assets = new List<Asset>();
    	
    	public override void OnEndOfAlgorithm()
    	{
    		_s3state.UploadWarmupState(_assets);
    	}
    	
    	private class AssetState // must be serializable by Json.net
    	{
    		public decimal MySavedValue;
    	}
    	
    	private class Asset : S3State.Serializable
    	{
    		private readonly Security _security;
    		
    		public Asset(QCAlgorithm algo, string symbol)
    		{
    			_security = algo.AddEquity(symbol, Resolution.Minute);
    		}
    		
    		public string GetKey()
    		{
    			return _security.Symbol.ToString();
    		}
    		
    		public object SerializeState()
    		{
    			var state = new AssetState();
    			state.MySavedValue = 3.14m; //should come from your class, obviously
    			return state;
    		}
    		
    		public bool DeserializeState(object o)
    		{
    			var state = (AssetState)o;
    			//do something with state.MySavedValue here
    			return true;
    		}
    	}

        /* 
        *	New data arrives here.
        *	The "Slice" data represents a slice of time, it has all the data you need for a moment.	
        */ 
        public override void OnData(Slice data) 
        {
        	// slice has lots of useful information
        	TradeBars bars = data.Bars;
        	Splits splits = data.Splits;
        	Dividends dividends = data.Dividends;
        	
        	//Get just this bar.
        	TradeBar bar;
        	if (bars.ContainsKey("SPY")) bar = bars["SPY"];
        	
            if (!Portfolio.HoldStock) 
            {
                // place an order, positive is long, negative is short.
                // Order("SPY",  quantity);
                
                // or request a fixed fraction of a specific asset. 
                // +1 = 100% long. -2 = short all capital with 2x leverage.
                SetHoldings("SPY", 1);
                
                // debug message to your console. Time is the algorithm time.
                // send longer messages to a file - these are capped to 10kb
                Debug("Purchased SPY on " + Time.ToShortDateString());
                //Log("This is a longer message send to log.");
            }
        }
    }
}

using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Net;
using System.Runtime.Serialization;
using System.Security.Cryptography;
using System.Text;
using System.Xml.Linq;

namespace QuantConnect.Algorithm.CSharp
{
    sealed class S3Client
    {
        public string UserID { get; set; }
        public string Password { get; set; }
        public Uri BaseURL { get; set; }

        private void CheckParameters()
        {
            if (UserID == null || UserID == "")
                throw new ArgumentException("UserID not set");
            if (Password == null || Password == "")
                throw new ArgumentException("Password not set");
            if (BaseURL == null)
                throw new ArgumentException("BaseURL not set");
        }

        private static void WriteToRequestWorkaround(HttpWebRequest request, byte[] data)
        {
            var stream = request.GetRequestStream();
            stream.Write(data, 0, data.Length);
        }

        private static void ReadFromResponseWorkaround(HttpWebResponse response, byte[] data)
        {
            var stream = response.GetResponseStream();
            for (int count = 0; count < data.Length; )
            {
                int numRead = stream.Read(data, count, data.Length - count);
                if (numRead > 0)
                    count += numRead;
                else
                    break;
            }
        }

        private static void ReadFromResponseUntilLineWorkaround(HttpWebResponse response, out byte[] data)
        {
            var bytes = new List<byte>();

            var stream = response.GetResponseStream();
            for (;;)
            {
                int next = stream.ReadByte();
                if (next == '\n')
                    break;

                bytes.Add((byte)next);
            }

            data = bytes.ToArray();
        }

        private HttpWebResponse Request(string method, string pathName, string objectName, byte[] content)
        {
            //ugly code follows, copied it from a buggy online Stack Overflow help request and fixed it

            string hash = "";
            if (content != null)
            {
                using (var md5 = new MD5CryptoServiceProvider())
                {
                    hash = Convert.ToBase64String(md5.ComputeHash(content));
                }
            }

            string contentType = "application/octet-stream";
            string date = DateTime.UtcNow.ToString("yyyyMMddTHHmmssZ");
            string file = "/" + pathName + "/" + objectName;

            //Creating signature
            var sBuilder = new StringBuilder();
            sBuilder.Append(method).Append("\n");
            if (content != null)
            {
                sBuilder.Append(hash).Append("\n");
                sBuilder.Append(contentType);
            }
            else
            {
                sBuilder.Append("\n");
            }

            sBuilder.Append("\n\n");
            sBuilder.Append("x-amz-date:").Append(date).Append("\n");
            sBuilder.Append(file);
            string stringToSign = sBuilder.ToString();
            //Console.WriteLine(stringToSign);
            var signature = Convert.ToBase64String(new HMACSHA1(Encoding.UTF8.GetBytes(Password)).ComputeHash(Encoding.UTF8.GetBytes(stringToSign)));

            var uri = new Uri(BaseURL, file);
            var request = WebRequest.Create(uri) as HttpWebRequest;
            request.Method = method;
            if (content != null)
            {
                request.ContentType = contentType;
                request.ContentLength = content.Length;
            }

            var headers = request.Headers;
            headers.Add("x-amz-date", date);
            if (content != null)
                headers.Add("Content-MD5", hash);
            headers.Add("Authorization", string.Format("AWS {0}:{1}", UserID, signature));

            if (content != null)
            {
                try
                {
                    WriteToRequestWorkaround(request, content);
                }
                catch (WebException e)
                {
                    Console.WriteLine(e.Message);
                    return null;
                }
            }

            HttpWebResponse response;
            try
            {
                response = request.GetResponse() as HttpWebResponse;
            }
            catch (WebException e)
            {
                Console.WriteLine(e.Message);
                response = e.Response as HttpWebResponse;
            }

            return response;
        }

        private class SerializationBinderWithoutAssembly : SerializationBinder
        {
            public override void BindToName(Type serializedType, out string assemblyName, out string typeName)
            {
                assemblyName = null;
                typeName = serializedType.FullName;
            }

            public override Type BindToType(string assemblyName, string typeName)
            {
                return Type.GetType(typeName);
            }
        }

        private readonly JsonSerializerSettings _jsonSettings = new JsonSerializerSettings
        {
            TypeNameHandling = TypeNameHandling.Auto,
            Binder = new SerializationBinderWithoutAssembly()
        };

        public bool GetObject<T>(string pathName, string objectName, out T data)
        {
            data = default(T);

            byte[] bytes;
            if (!GetObject(pathName, objectName, out bytes))
                return false;

            data = JsonConvert.DeserializeObject<T>(Encoding.UTF8.GetString(bytes), _jsonSettings);
            return true;
        }

        public bool GetObject(string pathName, string objectName, out byte[] data)
        {
            data = null;

            CheckParameters();

            var response = Request("GET", pathName, objectName, null);

            if (response == null)
                return false;

            if (response.StatusCode == HttpStatusCode.OK)
            {
                data = new byte[(int)response.ContentLength];
                ReadFromResponseWorkaround(response, data);

                return true;
            }

            var content = new byte[0];
            ReadFromResponseUntilLineWorkaround(response, out content);
            string responseContent = Encoding.UTF8.GetString(content);

            //var xmlResponse = XDocument.Parse(responseContent);
            Console.WriteLine(response.StatusCode);
            Console.WriteLine(responseContent);

            //TODO: extract error message from response

            return false;
        }

        public bool PutObject(string pathName, string objectName, object data)
        {
            var bytes = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(data, _jsonSettings));
            return PutObject(pathName, objectName, bytes);
        }

        public bool PutObject(string pathName, string objectName, byte[] data)
        {
            CheckParameters();

            var response = Request("PUT", pathName, objectName, data);

            if (response == null)
                return false;

            if (response.StatusCode == HttpStatusCode.OK
                || response.StatusCode == HttpStatusCode.Accepted)
                return true;

            var content = new byte[0];
            ReadFromResponseUntilLineWorkaround(response, out content);
            string responseContent = Encoding.UTF8.GetString(content);

            //var xmlResponse = XDocument.Parse(responseContent);
            Console.WriteLine(response.StatusCode);
            Console.WriteLine(responseContent);

            //TODO: extract error message from response

            return false;
        }
    }
}

using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;

namespace QuantConnect.Algorithm.CSharp
{
    sealed class S3State
    {
        public interface Serializable
        {
            string GetKey();
            object SerializeState();
            bool DeserializeState(object o);
        }

        private readonly QCAlgorithm _algo;
        private readonly S3Client _s3Client;
        public string AlgoID
        {
            get; set;
        }
        public string Bucket
        {
            get; set;
        }

        public S3State(QCAlgorithm algo, S3Client client)
        {
            _algo = algo;
            _s3Client = client;

            AlgoID = "untitled";
            Bucket = "examplebucket";
        }

        public void UploadWarmupState(IEnumerable<Serializable> objects)
        {
            if (_algo.LiveMode)
                return; //shouldn't be true from where it's called but check anyway
            //reasoning is that we might have several live versions of algo running and it's better to warm up from controlled backtest

            Dictionary<string, object> states;
            if (_s3Client.GetObject(Bucket, AlgoID + ".js", out states))
            {
                object lastDateObject;
                if (states.TryGetValue("LastDate", out lastDateObject) && lastDateObject is DateTime)
                {
                    var lastDate = (DateTime)lastDateObject;
                    if (_algo.Time.Date < lastDate.Date)
                    {
                        Print("Not uploading serialized state, there is already a newer state");
                        return;
                    }
                }
            }

            states = new Dictionary<string, object>();
            states["LastDate"] = _algo.Time.Date;

            foreach (var tradable in objects)
            {
                string ID = tradable.GetKey();

                object tradableState = tradable.SerializeState();
                if (tradableState == null)
                {
                    Print("Not uploading serialized state, too little warm up time for " + ID);
                    return;
                }

                states[ID] = tradableState;
            }

            if (_s3Client.PutObject(Bucket, AlgoID + ".js", states))
                Print("Uploaded serialized state to " + Bucket);
            else
                Print("Unable to upload serialized state to " + Bucket);
        }

        public void DownloadWarmupState(IEnumerable<Serializable> objects)
        {
            Dictionary<string, object> states;
            if (!_s3Client.GetObject(Bucket, AlgoID + ".js", out states))
            {
                string msg = "Unable to download warmup state from " + Bucket;
                if (_algo.LiveMode)
                    throw new Exception(msg);
                else
                    _algo.Debug(msg);

                return;
            }

            object lastDateObject;
            if (states.TryGetValue("LastDate", out lastDateObject) && lastDateObject is DateTime)
            {
                var lastDate = (DateTime)lastDateObject;
                double daysDifference = (_algo.Time.Date - lastDate.Date).TotalDays;
                if (daysDifference > 4)
                {
                    string msg = "Algo state on " + Bucket + " seems very old (" + daysDifference + " days)";
                    if (_algo.LiveMode)
                        throw new Exception(msg);
                    else
                        _algo.Debug(msg);
                }
                else if (daysDifference < -1 && !_algo.LiveMode)
                {
                    string msg = "Algo state on " + Bucket + " is in the future (" + -daysDifference + " days), skipping";
                    Print(msg);
                    return;
                }
            }

            states.Remove("LastDate");

            var tradablesRemaining = new List<Serializable>();
            tradablesRemaining.AddRange(objects);

            foreach (var kv in states)
            {
                var tradable = tradablesRemaining.Find(x => x.GetKey() == kv.Key);
                if (tradable != null)
                {
                    if (tradable.DeserializeState(kv.Value))
                        tradablesRemaining.Remove(tradable);
                }
            }

            if (tradablesRemaining.Count > 0)
            {
                string msg = "There wasn't correct algo state to deserialize for all tradables, e.g. " + tradablesRemaining[0].GetKey();
                if (_algo.LiveMode)
                    throw new Exception(msg);
                else
                    _algo.Debug(msg);
            }

            Print("Downloaded warmup state from " + Bucket);
        }

        private void Print(string msg)
        {
            if (_algo.LiveMode)
                _algo.Log(msg);
            else
                _algo.Debug(msg);
        }
    }
}

using System;

namespace QuantConnect.Algorithm.CSharp
{
    //new(): because references are atomic, and we wish to support calling DownloadParameters() from another thread while using Parameters property
    sealed class S3ParamSet<ParamType> where ParamType : new()
    {
        private readonly QCAlgorithm _algo;
        private readonly S3Client _s3Client;

        public string ParamSetID
        {
            get; set;
        }
        public string Bucket
        {
            get; set;
        }
        public bool ThrowOnErrorBeforeOnceSuccessful
        {
            get; set;
        }

        public ParamType Parameters
        {
            get { return _params; }
        }

        private ParamType _params = new ParamType();

        private bool _lastSuccess = true;
        private bool _anySuccess = false;

        public S3ParamSet(QCAlgorithm algo, S3Client client)
        {
            _algo = algo;
            _s3Client = client;

            ThrowOnErrorBeforeOnceSuccessful = true;
            ParamSetID = "config";
            Bucket = "examplebucket";
        }

        public bool DownloadParameters()
        {
            string filename = ParamSetID;
            if (_algo.LiveMode)
                filename += ".live";
            else
                filename += ".test";
            filename += ".js";
            string path = Bucket + "/" + filename;

            ParamType newParams;
            if (_s3Client.GetObject(Bucket, filename, out newParams))
            {
                _params = newParams;
                _lastSuccess = true;
                _anySuccess = true;
            }
            else
            {
                if (_lastSuccess)
                {
                    string msg = "Unable to download parameters from " + path;
                    if (ThrowOnErrorBeforeOnceSuccessful && !_anySuccess)
                        throw new Exception(msg);
                    else
                        _algo.Error(msg);
                }

                _lastSuccess = false;
            }

            return _lastSuccess;
        }
    }
}