/*
	This program was developed by Quantify and is a template program.
	Usage and marketing of this program is permitted.
	
	www.quantify-co.com
*/

namespace QuantConnect.Algorithm.CSharp
{
    public class i_kamanu3 : QCAlgorithm
    {
    	// BACKTESTING PARAMETERS
    	// =================================================================================================================
    	
    	// general settings:
    	
    	// set starting cash
    	private int starting_cash = 100000;
    	
    	// backtesting start date time:
    	// date setting variables
    	private int start_year = 2021;
    	private int start_month = 5;
    	private int start_day = 1;
    	
    	// backtesting end date time:
    	// determines whether there is a specified end date
    	// if false it will go to the current date (if 'true' it will go to the specified date)
    	private bool enable_end_date = false;
    	// date setting variables
    	private int end_year = 2021;
    	private int end_month = 1;
    	private int end_day = 1;
    	
    	
    	// universe settings:
    	
    	// number of symbols you want to be observed by the universe at any given time
    	// updates based on the universe resolution set
    	// recommended universe resolution is daily
    	private int stockCount = 15;
    	
    	// data update resolution
    	// changes how often the data updates and algorithm looks for entry
    	// determines how often the function OnData runs
    	// list of resolutions:
    	// Resolution.Tick; Resolution.Second; Resolution.Minute; Resolution.Hour; Resolution.Daily
    	private readonly Resolution resolution = Resolution.Hour;
    	
    	// portfolio allocation
		// percent of portfolio to allocate to overall (evenly divided between securities)
		private readonly decimal portfolioAllocation = 0.5m;
    	
    	// API settings:
    	
    	// NASDAQ API key:
    	private readonly string apiKey = "9xMzJxNYmHaXix2xTpKt";
    	
    	// Data entry from call:
    	// see documentation for details: https://data.nasdaq.com/data/VOL-us-equity-historical-option-implied-volatilities/documentation
    	private readonly string apiDataEntry = "IvMean90";
    	
    	
    	// algorithm parameters:
    	
		// high IV select (selects top n% of stocks with the highest IV values)
    	// note this value is in decimal form: 0.5 = 50%
    	private readonly decimal highIVSelection = 0.5m;
    	
    	// drawdown percentile
    	// percent of current IV based on the 12 month historical high IV of a security
    	// note this value is in decimal form: 0.5 = 50%
    	private readonly decimal securityIVPercentile = 0.5m;
    	
    	// =================================================================================================================
		
		// creates new universe variable setting
		private List<StockData> universe = new List<StockData>();
		
		// security changes variable
		private SecurityChanges securityChanges = SecurityChanges.None;
		
		// connection object
		private Connection connection;
		
		// month of the current universe
		private int currentMonth = 0;
		
		// determines if universe changed
		private bool load = false;
		
        public override void Initialize()
        {
        	// set start date
            SetStartDate(start_year, start_month, start_day);
            // set end date
            if(enable_end_date)
            	SetEndDate(end_year, end_month, end_day);
            // set starting cash
            SetCash(starting_cash);
            
            // add coarse selection for universe
            AddUniverse(CoarseFilterFunction);
            
            // schedule data load for Monday morning before market open
            Schedule.On(DateRules.Every(DayOfWeek.Monday, DayOfWeek.Tuesday, DayOfWeek.Wednesday, DayOfWeek.Thursday, DayOfWeek.Friday), 
            		TimeRules.At(09, 30), LoadSymbols);
            
            // define connection
            connection = new Connection(this, apiKey, apiDataEntry);
        }

		// filter based on CoarseFundamental
	    public IEnumerable<Symbol> CoarseFilterFunction(IEnumerable<CoarseFundamental> coarse) {
	    	// check if it is the first of the month, otherwise return current universe
	    	if(Time.Month == currentMonth) {
	    		return Universe.Unchanged;
	    	}
	    	
	    	currentMonth = Time.Month;
	    	load = true;
	    	// returns the highest DollarVolume stocks
	    	// returns "totalNumberOfStocks" amount of stocks
	        return (from stock in coarse
        			orderby stock.DollarVolume descending
        			select stock.Symbol).Take(stockCount);
	    }

        // OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
        // Slice object keyed by symbol containing the stock data
        private List<StockData> buffer = new List<StockData>();
        public override void OnData(Slice data) {
        	// if no securities in buffer then return
        	if(buffer.Count() == 0)
        		return;
        		
        	// set holdings for all securities in buffer than meet conditional
        	foreach(StockData sd in buffer)
        		if(sd.percentile < securityIVPercentile && !Securities[sd.ticker].Invested)
        			SetHoldings(sd.ticker, portfolioAllocation / buffer.Count());
        			
        	// clear buffer
        	buffer.Clear();
        }
        
        // Loads all Historical IV data into the symbols for the month
        public void LoadSymbols() {
        	if(!load)
        		return;
        	
        	// load each symbol with historical IV data (note last value is current IV)
        	foreach(StockData sd in universe) {
        		sd.iv = connection.Load(sd.parsed, Time);
        		
        		// if no elements then submit error
        		if(sd.iv.Count() == 0) {
        			Error($"No elements recorded for: {sd.parsed}");
        			continue;
        		}
        		
        		// get and remove last element
        		sd.current = sd.iv[sd.iv.Count() - 1];
        		sd.iv.Remove(sd.iv.Count() - 1);
        		
        		// find max
        		decimal max = sd.iv[0];
        		foreach(decimal d in sd.iv)
        			if(d > max)
        				max = d;
        		
        		// determines if the maximum is faulty
        		if(max == 0.0m)
        			sd.percentile = Decimal.MaxValue;
        		else
        			sd.percentile = sd.current / max;
        		
        		// calculate percentile based on maximum iv
        		//Debug($"Percentile - {sd.parsed} = {sd.percentile} [{sd.current} / {max}]");
        	}
        	
        	// selection amount
        	int n1 = Decimal.ToInt32(Math.Ceiling(highIVSelection * stockCount));
        	var highIVSecurities = (from sd in universe
        				orderby sd.current descending
        				select sd).Take(n1);
        	
        	// push all securities to the buffer
        	foreach(StockData sd in highIVSecurities)
        		buffer.Add(sd);
        		
        	// update universe to loaded status
        	load = false;
        }
        
        // OnSecuritiesChanged runs when the universe updates current securities
        public override void OnSecuritiesChanged(SecurityChanges changes) {
            securityChanges = changes;
            // remove stocks from list that get removed from universe
            foreach (var security in securityChanges.RemovedSecurities) {
            	List<StockData> stockDatas = universe.Where(x=>x.ticker == security.Symbol).ToList();
            	if (stockDatas.Count >= 1) {
            		// check to see if position is open and if so close position
            		if(Portfolio[stockDatas.First().ticker].Invested) {
            			// closes position
            			Liquidate(stockDatas.First().ticker);
            			Log($"Liquidated {stockDatas.First().parsed} on removal.");
            			
            		}
                	Log($"Removed {stockDatas.First().parsed} from universe.");
            		
            		// removes stock from list if it is removed from the universe
                	universe.Remove(stockDatas.First());
            	}
            }
            
            // add new securities to universe list
            foreach(var security in securityChanges.AddedSecurities) {
            	// create StockData variable for security
            	StockData sd = new StockData();
            	sd.ticker = security.Symbol;
            	// removes QC code
            	sd.parsed = sd.ticker.Split(' ')[0];
    			sd.iv = new List<Decimal>();
    			sd.percentile = Decimal.MaxValue;
    			
            	// add stockdata to universe
            	universe.Add(sd);
            	
            	Log($"Added {sd.parsed} to universe.");
            }
        }
		
		// default class containing all ticker information
		public class StockData {
			// stock ticker
			public string ticker = "";
			public string parsed = "";
			
			// historical IV values
			public List<Decimal> iv;
			
			// current iv value
			public decimal current;
			// iv percentile variables
			public decimal percentile;
		}
    }
}

namespace QuantConnect {

	public class Connection {
		
		private readonly QCAlgorithm algorithm;
		
		// sample API call for MSFT on 2021-01-05 as a reference:
		//"https://data.nasdaq.com/api/v3/datasets/VOL/MSFT/data?start_date=2021-01-05&end_date=2021-01-05&api_key=9xMzJxNYmHaXix2xTpKt"
		
		// base url for API calls
		private readonly string url = "https://data.nasdaq.com/api/v3/datasets/VOL/";
		// API to use in call
		private readonly string key;
		// data point entry to look for in call
		private readonly string entry;
		// index of the entry within the call (so we don't scan for it every time)
		private int entryIndex = -1;
		
		// hash of the current dates stored so they don't have to be refreshed
		private string hash = "null";
		// list of the dates stored under the current hash
		private List<DateTime> dates = new List<DateTime>();
		
		// Connection constructor to import the QCAlgorithm, API key, and entry name
		public Connection(QCAlgorithm algorithm, string key, string entry) {
			this.algorithm = algorithm;
			this.key = key;
			this.entry = entry;
		}
		
		// used to retrieve a list of all historical IV values for the prior year
		// calls the object cache should the value already exist for that month
		public List<Decimal> Load(string symbol, DateTime time) {
			// cross reference hash with current time
			// if different then load new dates
			if(hash != $"{time.Year}{time.Month}{time.Day}") {
				hash = $"{time.Year}{time.Month}{time.Day}";
				
				// get all historical dates
				dates = new List<DateTime>();
				DateTime start = new DateTime(time.Year - 1, time.Month, time.Day);
				DateTime end = time;
				for(var dt = start; dt <= end; dt = dt.AddMonths(1))
					dates.Add(dt);
			}
			
			// request each date and add it to a list
			List<Decimal> values = new List<Decimal>();
			foreach(DateTime dt in dates) {
				decimal requested = Request(symbol, dt.Year, dt.Month, dt.Day, time);
				// if invalid date
				if(requested == -1.0m)
					algorithm.Log($"Warning: Unable to retrieve IV for {symbol} on {dt.Year}-{dt.Month}-{dt.Day}.");
				else
					values.Add(requested);
			}
			return values;
		}
		
		// requests the historical IV value for the given date
		// takes the symbol, date, and current date as parameters
		private decimal Request(string symbol, int year, int month, int day, DateTime bounds) {
			// check for cache existing
			string key = Hash(symbol, entry, year, month);
			// if it exists then return stored value
			if (algorithm.ObjectStore.ContainsKey(key)) {
				decimal stored = Convert.ToDecimal(algorithm.ObjectStore.Read(key));
				Print($"Read hash for: {key} = {stored}");
				return stored;
			}
			
			// download and parse data from URL request
			Print($"Requesting: {Format(symbol, year, month, day)}");
			string data = algorithm.Download(Format(symbol, year, month, day));
			// parse by line
			string[] split = data.Split('\n');
			Console.WriteLine(String.Join(",", split));
			
			// if only 2 lines then request the next day
			// if past the DateTime bound then return -1
			if(split.Count() <= 2) {
				DateTime dt;
				// make sure testing valid date
				bool valid = DateTime.TryParse($"{month}/{day + 1}/{year}", out dt);
				
				// if invalid date or out of bounds, return -1
				if(!valid || dt > bounds) {
					Print($"Invalid: {year}-{month}-{day+1} :: {dt > bounds}");
					return -1;
				}
				
				Print($"Requesting Next Day: {year}-{month}-{day+1}");
				// request next day
				return Request(symbol, year, month, day + 1, bounds);
			}
			
			// if the entry index has not been parsed yet, find the index of the value
			if(entryIndex == -1) {
				// parse columns
				string[] columns = split[0].Split(',');
				
				// look for column with matching name
				for(int i = 0; i < columns.Count(); i++) {
					if(columns[i] == entry) {
						entryIndex = i;
						break;
					}
				}
			}
			
			// parse value from data
			string[] values = split[1].Split(',');
			if(values.Count() <= entryIndex) {
				algorithm.Error($"Data issue, [entry points={values.Count()}]");
				return -1.0m;
			}
			//Console.WriteLine(String.Join(",", values));
			decimal value = 0.1m;
			//decimal value = Convert.ToDecimal(values[entryIndex]);
			Print($"Found Value: {year}-{month}-{day} {value}");
			
			// cache found value
			//Console.Write($"Created hash: {Hash(symbol, entry, year, month)}");
			algorithm.ObjectStore.Save(Hash(symbol, entry, year, month), $"{value}");
			
			return value;
		}
		
		private string Format(string symbol, int year, int month, int day) {
			string formatted = url;
			formatted += $"{symbol}/data.csv?start_date={year}-{month}-{day}" +
					$"&end_date={year}-{month}-{day}&api_key={key}";
			return formatted;
		}
		
		private string Hash(string symbol, string entry, int year, int month) {
			return $"{symbol}-{entry}:{year}-{month}";
		}
		
		private void Print(Object obj) {
			//Console.Write(obj);
		}
	}
}