using MathNet.Numerics;
using MathNet.Numerics.Statistics;
using System;
using System.Collections.Concurrent;
using System.Linq;
using QuantConnect.Data.Market;
using QuantConnect.Data.UniverseSelection;
using QuantConnect.Indicators;
using QuantConnect.Securities;
using System.Collections.Generic;
using QuantConnect.Data;
using QuantConnect;
using QuantConnect.Data.Fundamental;
using QuantConnect.Brokerages;
using QuantConnect.Algorithm.CSharp.Helpers;

namespace QuantConnect.Algorithm.CSharp
{
    /*
    Momentum strategy according to Andreas F. Clenow book 
    'Stocks on the Move Beating the Market with Hedge Fund Momentum Strategies'
    
    ** Objective of the algorithm: **
    
    The algorithm selects from the SP500 universe the stocks with fastest rising stocks with limited volatility.
    Every week the risk of the portfolio is balanced. Once a month the stocks are rebalanced.
    The algo only buys if the market is in a high trend (Bulish);
    
    ** Trade rules: **
    
    The rules are:
    - Rule 00: SP500 stocks
    - Rule 01: Trade on Wednesdays to limit number of trades;
    - Rule 02: Twice a month reset the positions sizes (risk);
    - Rule 03: Rebalance every week;
    - Rule 04: If the stock is not in the top 100/ 20% ranking, sell it;
    - Rule 05: If the stock is below its 100 days moving average, sell it;
    - Rule 06: If the stock gapped > 15%, sell it;
    - Rule 07: If the stock left the index, sell it;
    - Rule 08: If the SP500 is above the 200 days moving average we buy stocks, otherwise not;
    - Rule 09: Calculate the position sizes, based on 10 basis points using ATR formula;
    - Rule 10: Momentum is calculated based on 90 past days annualized exponential regression slope;
    - Rule 11: Momentum is weighted for volatility adjustment (r^2);
    - Rule 12: Position Size = (portfolioSize*0,001/ ATR)= #Shares;
    - Rule 13: Trade maximum 30 stocks;
	
	** Change history: **
    20180102_01: QT, created algo based on cloned algorithm. Refactoring algo.
    20180103_01: QT, seems that universe selecting may not have own calculations.
	*/
    /// <summary>
    /// Momentum strategy according to Andreas F. Clenow book 
    /// Stocks on the Move Beating the Market with Hedge Fund Momentum Strategies'	
    /// </summary>
    public class MomentumStrategyStockUniverseAlgorithm : QCAlgorithm
    {
        private const string SP500IndexSymbol = "SPY";

        /// Rule 00: SP500 stocks
        private static int _universeSelectMaxStocks = 50;
        private static bool _rebalanceFlag = false;

        /// Rule 10: Momentum is calculated based on 90 past days annualized exponential regression slope;
        private const int MomentumWindow = 90; 			
        // Rule 05: If the stock is below its 100 days moving average, sell it;
        private const int StockMovingAverageWindow = 100;// 100; 

        private Symbol _spy = QuantConnect.Symbol.Create(SP500IndexSymbol, SecurityType.Equity, Market.USA);
        private SimpleMovingAverage _spyMovingAverage;
        private AnnualizedExponentialSlopeIndicator _spyAnnualizedExponentialSlopeIndicator;
        private AnnualizedExponentialSlopeIndicator _sttAnnualizedExponentialSlopeIndicator;
        private decimal _spyPriceClose = 0;
        private decimal _sttPriceClose = 0;

        private SecurityChanges _securityChanges = SecurityChanges.None;
        // holds our coarse fundamental indicators by symbol
        private static readonly ConcurrentDictionary<Symbol, MomentumSelectionData> _momentums = new ConcurrentDictionary<Symbol, MomentumSelectionData>();

        // Rule 08: If the SP500 is above the 200 days moving average we buy stocks, otherwise not;
        private int _trendfilter = 200;

        // Rule 04: If the stock is not in the top 100/ 20% ranking, sell it;
        private int _topNStockOfSp500 = 100;
        // Rule 06: If the stock gapped > 15%, sell it;
        private decimal _stockMaximumgap = 0.15m;
        // Look back period of stock gap
        private int _stockMaximumGapWindow = 60;
        // Rule 13: Trade maximum 30 stocks;
        private int _maxStockInPortfolio = 30;

        private List<string> _stocksToTrade = new List<string>();

        private bool _isDebugging = true;
        private bool _isPlotting = true;
        private bool _isPlotSpyMovingAverage = true;
        private int _isLogSpyMovingAveragePivot = 0;

        // class used to improve readability of the coarse selection function
        // href https://www.quantconnect.com/forum/discussion/1233/portfolio-optimization-with-mathnet-numerics
        private class MomentumSelectionData
        {
            public readonly AnnualizedExponentialSlopeIndicator AnnualizedSlope;
            public readonly ExponentialMovingAverage MovingAverage;

            public MomentumSelectionData(int AnnualizedSlopeWindow, int movingAverageWindow)
            {
                AnnualizedSlope = new AnnualizedExponentialSlopeIndicator(AnnualizedSlopeWindow);
                MovingAverage = new ExponentialMovingAverage(movingAverageWindow);
            }

            // updates the indicators, returning true when they're both ready
            public bool Update(DateTime time, decimal value)
            {
                return AnnualizedSlope.Update(time, value) && MovingAverage.Update(time, value);
            }
        }
        /// <summary>
        /// Helper to create AnnualizedExponentialSlopeIndicator
        /// </summary>
        /// <param name="symbol">symbol</param>
        /// <param name="period">period</param>
        /// <param name="resolution">resolution of data</param>
        /// <returns></returns>
        public AnnualizedExponentialSlopeIndicator AESI(string symbol, int period, Resolution? resolution = null, Func<IBaseData, decimal> selector = null)
        {
            var name = CreateIndicatorName(symbol, string.Format("AESI({0})", period), resolution);
            var aesi = new AnnualizedExponentialSlopeIndicator(name, period);
            RegisterIndicator(symbol, aesi, resolution, selector);
            return aesi;
        }
        /// <summary>
        /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
        /// </summary>
        public override void Initialize()
        {
            //speed up execution
            if (IsDebugging)
            {
                _isPlotSpyMovingAverage = true;
                _universeSelectMaxStocks = 20;
                _trendfilter = 100;
                _topNStockOfSp500 = 20;
                _stockMaximumGapWindow = 14;
                _maxStockInPortfolio = 5;
            }

            //Set trading window
            SetStartDate(2016, 1, 1);
            SetEndDate(2018, 1, 1);
            //SetEndDate(DateTime.Now);

            //Set benchmark
            SetBenchmark("SPY");

            //Set brokermodel
            SetCash(100000);
            SetBrokerageModel(BrokerageName.InteractiveBrokersBrokerage, AccountType.Margin);

            //Set moving average on SPY and benchmark
            _spy = AddEquity(SP500IndexSymbol, Resolution.Daily).Symbol;
            _spyMovingAverage = SMA(SP500IndexSymbol, _trendfilter, Resolution.Daily);
            _spyAnnualizedExponentialSlopeIndicator = AESI(SP500IndexSymbol, MomentumWindow, Resolution.Daily);

            ////Set STT as benchmark
            //AddEquity("STT", Resolution.Daily);
            //_sttAnnualizedExponentialSlopeIndicator = AESI("STT", MomentumWindow, Resolution.Daily);

            //set warm up algorithm to avoid premature trading
            SetWarmUp(_trendfilter + 1);

            //setup universe with filters for coarse and fine: https://www.quantconnect.com/docs#Universes
            UniverseSettings.Resolution = Resolution.Daily;
            AddUniverse(CoarseSelectionFunction, FineSelectionFunction);

            //trade only on wednesdays at opening after 10 minutes
            Schedule.On(DateRules.Every(DayOfWeek.Wednesday, DayOfWeek.Wednesday),
                TimeRules.AfterMarketOpen(SP500IndexSymbol, 10), () =>
                {
                    _rebalanceFlag = true;  // Rebalance will create a new universe
                                            //_.bi_weekly +=1 #to-do
                });

            if (IsDebugging)
            {
                Log("*** DEBUGGING: TAKE CARE OF PARAMETERS ***");
            }


        }

        /// Is debugging set, speed up processing
        public bool IsDebugging { get { return _isDebugging; } }
        /// Is plotting set
        public bool IsPlotting { get { return _isPlotting; } }

        // calculate the f-score
        /// <summary>
        /// https://www.quantconnect.com/forum/discussion/1677/an-example-of-fundamental-ranking
        /// </summary>
        private int FScore(
            decimal netincome,
            decimal operating_cashflow,
            decimal roa_current, decimal roa_past,
            decimal issued_current, decimal issued_past,
            decimal grossm_current, decimal grossm_past,
            decimal longterm_current, decimal longterm_past,
            decimal curratio_current, decimal curratio_past,
            decimal assetturn_current, decimal assetturn_past
            )
        {
            int fscore = 0;

            fscore += (roa_current > 0m ? 1 : 0);  // return on assets is positive
            fscore += (operating_cashflow > 0m ? 1 : 0);  // operating cashflow in current year is positive
            fscore += (roa_current >= roa_past ? 1 : 0);  // roa has increased since last Time
            fscore += (operating_cashflow > roa_current ? 1 : 0);  // cashflow from current operations are greater than roa
            fscore += (longterm_current <= longterm_past ? 1 : 0);  // a decrease in the long term debt ratio
            fscore += (curratio_current >= curratio_past ? 1 : 0);  // an increase in the current ratio
            fscore += (issued_current <= issued_past ? 1 : 0);  // no new shares have been issued
            fscore += (grossm_current >= grossm_past ? 1 : 0);  // an increase in gross margin
            fscore += (assetturn_current >= assetturn_past ? 1 : 0); // a higher asset turnover ratio

            return fscore;
        }
        /// <summary>
        /// OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
        /// </summary>
        /// <param name="data">Slice object keyed by symbol containing the stock data</param>
        public override void OnData(Slice data)
        {
            if (!data.ContainsKey(SP500IndexSymbol))
            {
                Log(string.Format("!data.ContainsKey(SP500IndexSymbol) {0} ", SP500IndexSymbol));
                return;
            }
            _spyPriceClose = data[SP500IndexSymbol].Close;

            //if (!data.ContainsKey("STT"))
            //{
            //    Log(string.Format("!data.ContainsKey(STT) {0} ", "STT"));
            //    return;
            //}
            //_sttPriceClose = data["STT"].Close;

            if (IsWarmingUp) return;

            if (_rebalanceFlag)
            {
                Debug("_rebalanceFlag = true");
            }
            //Rule 01: Trade on Wednesdays to limit number of trades;
            if (Time.DayOfWeek != DayOfWeek.Wednesday)
            {
                return;
            }

            #region SecurityChanges
            // if we have no changes, do nothing
            if (_securityChanges != SecurityChanges.None)
            {
                // Liquidate removed securities that do not rank anymore
                // Rule 07: If the stock left the index, sell it;
                foreach (var security in _securityChanges.RemovedSecurities)
                {
                    if (security.Invested)
                    {
                        Liquidate(security.Symbol);
                    }
                }
            }
            //not in use
            foreach (var security in _securityChanges.AddedSecurities)
            {
                Log(string.Format("_securityChanges.AddedSecurities, Security: {0} ", security));
            }

            #endregion SecurityChanges

            //#- Rule 10: Momentum is calculated based on self.momentum_window (90) past days annualized exponential regression slope;
            //#- Rule 11: Momentum is weighted for volatility adjustment (r^2);
            //stock_dict = {}
            List<Security> buyList = new List<Security>(30);
            Log(string.Format("Securities.Count: {0}", Securities.Count.ToString()));
            foreach (var security in Securities)
            {

                //- Rule 10: Momentum is calculated based on self.momentum_window (90) past days annualized exponential regression slope;
                //- Rule 11: Momentum is weighted for volatility adjustment (r^2);
                //Spy is not tradeble
                if (string.Compare(security.Key.Value, SP500IndexSymbol, true) == 0)
                    continue;

                Log(string.Format("Securities (unsorted) -> Security: {0} ", security));

                MomentumSelectionData stockMomentumSelectionData = null;
                if (_momentums.TryGetValue(security.Value.Symbol, out stockMomentumSelectionData))
                {
                    if (!stockMomentumSelectionData.MovingAverage.IsReady || !stockMomentumSelectionData.AnnualizedSlope.IsReady)
                        continue;

                    Log(string.Format("{0}: {1} {2}", security.Value.Symbol, stockMomentumSelectionData.AnnualizedSlope.ToDetailedString(), stockMomentumSelectionData.MovingAverage.ToDetailedString()));

                    //- Rule 10: Momentum is calculated based on 90 past days annualized exponential regression slope;
                    decimal symbolMomentum = stockMomentumSelectionData.AnnualizedSlope.Current.Value;
                    if(symbolMomentum < 0)
                    {
                        Log(string.Format("{0}: AnnualizedSlope: {1} #- Rule 10: If the stock is below its {2} days momentum, sell it;",
                                    security.Value.Symbol, symbolMomentum.ToString(), MomentumWindow));
                        continue;
                    }
                    //- Rule 05: If the stock is below its 100 days moving average, remove it;
                    decimal symbolMovingAverage = stockMomentumSelectionData.MovingAverage.Current.Value;
                    if (security.Value.Price < symbolMovingAverage)
                    {
                        Log(string.Format("{0}: {1}<{2} #- Rule 05: If the stock is below its {3} days moving average, sell it;",
                                    security.Value.Symbol, security.Value.Price, symbolMovingAverage, StockMovingAverageWindow));
                        continue;
                    }

                    //- Rule 06: If the stock gapped > 15%, remove it
                    decimal symbolMaximumGap = GetGap(security.Value.Symbol, _stockMaximumGapWindow);
                    if (symbolMaximumGap > _stockMaximumgap)
                    {
                        Log(string.Format("{0}: {1}>{2} #- Rule 06: If the stock gapped > {3}% in window lookback {4}, sell it",
                            security.Value.Symbol, symbolMaximumGap, _stockMaximumgap, _stockMaximumgap * 100, _stockMaximumGapWindow));
                        continue;
                    }
                    Log(string.Format("{0} Gap: {1}", security.Value.Symbol, symbolMaximumGap.ToString()));
                    Log(string.Format("{0} added to buylist", security.Value.Symbol));
                    buyList.Add(security.Value);
                }

            }


            if (Portfolio.Invested)
            {
                //clean up portfolio
                foreach (SecurityHolding stock in Portfolio.Values)
                {
                    if (stock.Invested)
                    {
                        
                        Symbol symbol = stock.Symbol;
                        
                            //Log(string.Format("clean up portfolio, symbol: {0}" , symbol));
                            //- Rule 05: If the stock is below its 100 days moving average, sell it;
                            decimal symbolMovingAverage = GetMean(symbol, StockMovingAverageWindow);
                        if (Securities[symbol].Price < symbolMovingAverage)
                        {
                            Liquidate(symbol);
                            Log(string.Format("{0}: {1}<{2} #- Rule 05: If the stock is below its {3} days moving average, sell it;",
                                symbol, Securities[symbol].Price, symbolMovingAverage, StockMovingAverageWindow));
                        }
                        else
                        {
                            //- Rule 06: If the stock gapped > 15%, sell it
                            //symbolMaximumGap
                            decimal symbolMaximumGap = GetGap(symbol, _stockMaximumGapWindow);
                            if (symbolMaximumGap > _stockMaximumgap)
                            {
                                Liquidate(symbol);
                                Log(string.Format("{0}: {1}>{2} #- Rule 06: If the stock gapped > {3}% in window lookback {4}, sell it",
                                    symbol, symbolMaximumGap, _stockMaximumgap, _stockMaximumgap * 100, _stockMaximumGapWindow));
                            }
                        }
                    }
                }
            }
            //No securities are in the universe in case it is not a rebalance day
            if (!_rebalanceFlag)
            {
                return;
            }
            //- Rule 08: If the SP500 is above the 200 days moving average we buy stocks, otherwise not;
            if (Securities[_spy].Price <= _spyMovingAverage)
            {
                //$TODO: buy T-note/ gold/ silver?
                if (_isLogSpyMovingAveragePivot >= 0)
                {
                    Log(string.Format("Spy in downtrend: {0} < {1}", Securities[_spy].Price, _spyMovingAverage));
                    _isLogSpyMovingAveragePivot = -1;
                }
            }
            else
            {
                if (_isLogSpyMovingAveragePivot <= 0)
                {
                    Log(string.Format("Spy in uptrend: {0} > {1}", Securities[_spy].Price, _spyMovingAverage));
                    _isLogSpyMovingAveragePivot = 1;
                }
                foreach(Security security in buyList)
                {
                    SetHoldings(security.Symbol, 1m/buyList.Count);
                }
            }


            //stock_dict = {}

            ///Plotting
            if (IsPlotting)
            {

            }

            _rebalanceFlag = false;

            //indicate that_changes are processed
            _securityChanges = SecurityChanges.None;

        }

        /// Get the momentum over the lookback period
        /// Input:  Security symbol and the look back period
        ///	Output: Annualized exponential regression slope, multiplied by volatility
        /// 		adjusted momentum measurement (coefficient of determination)
        public decimal GetMomentum(Symbol symbol, int windowSize)
        {
            //validate that the security is in the universe
            if (!Securities.ContainsKey(symbol))
                return 0;

            IEnumerable<Slice> slices = History(windowSize, Resolution.Daily);
            var window = slices.Get(symbol, Field.Close).ToList();
            //var closes = window.ToDoubleArray();

            if (window.Count < 3) return 0m;

            var xVals = new double[window.Count];
            var yVals = new double[window.Count];

            // load input data for regression
            for (int i = 0; i < window.Count; i++)
            {
                xVals[i] = i;
                // we want the log of our y values
                yVals[i] = Math.Log((double)window[(window.Count - i - 1)]);
            }

            //http://numerics.mathdotnet.com/Regression.html

            // solves y=a + b*x via linear regression
            var fit = Fit.Line(xVals, yVals);
            var intercept = fit.Item1;
            var slope = fit.Item2;

            // compute rsquared
            var rsquared = GoodnessOfFit.RSquared(xVals.Select(x => intercept + slope * x), yVals);

            // anything this small can be viewed as flat
            if (double.IsNaN(slope) || Math.Abs(slope) < 1e-25) return 0m;

            // trading days per year for us equities
            const int dayCount = 252;

            // annualize dy/dt
            var annualSlope = ((Math.Pow(Math.Exp(slope), dayCount)) - 1) * 100;

            // scale with rsquared
            annualSlope = annualSlope * rsquared;

            return (decimal)annualSlope;

        }
        /// <summary> 
        /// Calculate Mean
        /// </summary>
        private decimal GetMean(Symbol symbol, int windowSize)
        {
            //validate that the security is in the universe
            if (!Securities.ContainsKey(symbol))
                return 0;

            IEnumerable<Slice> slices = History(windowSize, Resolution.Daily);
            IEnumerable<decimal> close = slices.Get(symbol, Field.Close);
            var closes = close.ToDoubleArray();

            return (decimal)(closes.Mean());
        }
        /// <summary> 
        /// Calculate Gap
        /// return np.max(np.abs(np.diff(close_data))/close_data[:-1])
        //  	out[n] = (a[n+1] - a[n]) / a[n]
        /// </summary>
        private decimal GetGap(Symbol symbol, int windowSize)
        {
            //validate that the security is in the universe
            if (!Securities.ContainsKey(symbol))
                return 0;

            IEnumerable<Slice> slices = History(windowSize, Resolution.Daily);
            var window = slices.Get(symbol, Field.Close).ToList();
            //var closes = close.ToDoubleArray();

            if (window.Count < 3) return 0m;

            var diff = new double[window.Count];

            // load input data for regression
            for (int i = 0; i < window.Count - 1; i++)
            {
                diff[i] = (double)((window[i + 1] - window[i]) / (window[i] == 0 ? 1 : window[i]));
            }

            return (decimal)diff.MaximumAbsolute();
        }
        // Select S&P500 stocks only:
        // Market capitalization must be greater than or equal to $6.1 billion USD
        // Annual dollar value traded to float-adjusted market capitalization is greater than 1.0
        // Minimum monthly trading volume of 250,000 shares in each of the six months leading up to the evaluation date
        public IEnumerable<Symbol> CoarseSelectionFunction(IEnumerable<CoarseFundamental> coarse)
        {
            //const decimal MarketCapitalization = 2100000000m; 
            //^not supported by Quantconnect;

            const decimal DollarVolume = 20000000m;

            if (!_rebalanceFlag) return Enumerable.Empty<Symbol>();

            //https://www.barchart.com/
            List<string> sp500StockList = new List<string>() { "A", "AAL", "AAP", "AAPL", "ABBV", "ABC", "ABT", "ACN", "ADBE", "ADI", "ADM", "ADP", "ADS", "ADSK", "AEE", "AEP", "AES", "AET", "AFL", "AGN", "AIG", "AIV", "AIZ", "AJG", "AKAM", "ALB", "ALGN", "ALK", "ALL", "ALLE", "ALXN", "AMAT", "AMD", "AME", "AMG", "AMGN", "AMP", "AMT", "AMZN", "ANDV", "ANSS", "ANTM", "AON", "AOS", "APA", "APC", "APD", "APH", "APTV", "ARE", "ARNC", "ATVI", "AVB", "AVGO", "AVY", "AWK", "AXP", "AYI", "AZO", "BA", "BAC", "BAX", "BBT", "BBY", "BDX", "BEN", "BF.B", "BHF", "BHGE", "BIIB", "BK", "BLK", "BLL", "BMY", "BRK.B", "BSX", "BWA", "BXP", "C", "CA", "CAG", "CAH", "CAT", "CB", "CBG", "CBOE", "CBS", "CCI", "CCL", "CDNS", "CELG", "CERN", "CF", "CFG", "CHD", "CHK", "CHRW", "CHTR", "CI", "CINF", "CL", "CLX", "CMA", "CMCSA", "CME", "CMG", "CMI", "CMS", "CNC", "CNP", "COF", "COG", "COL", "COO", "COP", "COST", "COTY", "CPB", "CRM", "CSCO", "CSRA", "CSX", "CTAS", "CTL", "CTSH", "CTXS", "CVS", "CVX", "CXO", "D", "DAL", "DE", "DFS", "DG", "DGX", "DHI", "DHR", "DIS", "DISCA", "DISCK", "DISH", "DLR", "DLTR", "DOV", "DPS", "DRE", "DRI", "DTE", "DUK", "DVA", "DVN", "DWDP", "DXC", "EA", "EBAY", "ECL", "ED", "EFX", "EIX", "EL", "EMN", "EMR", "EOG", "EQIX", "EQR", "EQT", "ES", "ESRX", "ESS", "ETFC", "ETN", "ETR", "EVHC", "EW", "EXC", "EXPD", "EXPE", "EXR", "F", "FAST", "FB", "FBHS", "FCX", "FDX", "FE", "FFIV", "FIS", "FISV", "FITB", "FL", "FLIR", "FLR", "FLS", "FMC", "FOX", "FOXA", "FRT", "FTI", "FTV", "GD", "GE", "GGP", "GILD", "GIS", "GLW", "GM", "GOOG", "GOOGL", "GPC", "GPN", "GPS", "GRMN", "GS", "GT", "GWW", "HAL", "HAS", "HBAN", "HBI", "HCA", "HCN", "HCP", "HD", "HES", "HIG", "HII", "HLT", "HOG", "HOLX", "HON", "HP", "HPE", "HPQ", "HRB", "HRL", "HRS", "HSIC", "HST", "HSY", "HUM", "IBM", "ICE", "IDXX", "IFF", "ILMN", "INCY", "INFO", "INTC", "INTU", "IP", "IPG", "IQV", "IR", "IRM", "ISRG", "IT", "ITW", "IVZ", "JBHT", "JCI", "JEC", "JNJ", "JNPR", "JPM", "JWN", "K", "KEY", "KHC", "KIM", "KLAC", "KMB", "KMI", "KMX", "KO", "KORS", "KR", "KSS", "KSU", "L", "LB", "LEG", "LEN", "LH", "LKQ", "LLL", "LLY", "LMT", "LNC", "LNT", "LOW", "LRCX", "LUK", "LUV", "LYB", "M", "MA", "MAA", "MAC", "MAR", "MAS", "MAT", "MCD", "MCHP", "MCK", "MCO", "MDLZ", "MDT", "MET", "MGM", "MHK", "MKC", "MLM", "MMC", "MMM", "MNST", "MO", "MON", "MOS", "MPC", "MRK", "MRO", "MS", "MSFT", "MSI", "MTB", "MTD", "MU", "MYL", "NAVI", "NBL", "NCLH", "NDAQ", "NEE", "NEM", "NFLX", "NFX", "NI", "NKE", "NLSN", "NOC", "NOV", "NRG", "NSC", "NTAP", "NTRS", "NUE", "NVDA", "NWL", "NWS", "NWSA", "O", "OKE", "OMC", "ORCL", "ORLY", "OXY", "PAYX", "PBCT", "PCAR", "PCG", "PCLN", "PDCO", "PEG", "PEP", "PFE", "PFG", "PG", "PGR", "PH", "PHM", "PKG", "PKI", "PLD", "PM", "PNC", "PNR", "PNW", "PPG", "PPL", "PRGO", "PRU", "PSA", "PSX", "PVH", "PWR", "PX", "PXD", "PYPL", "QCOM", "QRVO", "RCL", "RE", "REG", "REGN", "RF", "RHI", "RHT", "RJF", "RL", "RMD", "ROK", "ROP", "ROST", "RRC", "RSG", "RTN", "SBAC", "SBUX", "SCG", "SCHW", "SEE", "SHW", "SIG", "SJM", "SLB", "SLG", "SNA", "SNI", "SNPS", "SO", "SPG", "SPGI", "SRCL", "SRE", "STI", "STT", "STX", "STZ", "SWK", "SWKS", "SYF", "SYK", "SYMC", "SYY", "T", "TAP", "TDG", "TEL", "TGT", "TIF", "TJX", "TMK", "TMO", "TPR", "TRIP", "TROW", "TRV", "TSCO", "TSN", "TSS", "TWX", "TXN", "TXT", "UA", "UAA", "UAL", "UDR", "UHS", "ULTA", "UNH", "UNM", "UNP", "UPS", "URI", "USB", "UTX", "V", "VAR", "VFC", "VIAB", "VLO", "VMC", "VNO", "VRSK", "VRSN", "VRTX", "VTR", "VZ", "WAT", "WBA", "WDC", "WEC", "WFC", "WHR", "WLTW", "WM", "WMB", "WMT", "WRK", "WU", "WY", "WYN", "WYNN", "XEC", "XEL", "XL", "XLNX", "XOM", "XRAY", "XRX", "XYL", "YUM", "ZBH", "ZION", "ZTS"};

            // Market capitalization must be greater than or equal to $6.1 billion USD, Traded shares x Price
            var filtered = from x in coarse
                           join SP500 in sp500StockList on x.Symbol.Value equals SP500
                           let momentums = _momentums.GetOrAdd(x.Symbol, sym => new MomentumSelectionData(MomentumWindow, StockMovingAverageWindow))
                           // Update returns true when the indicators are ready, so don't accept until they are
                           where momentums.Update(x.EndTime, x.Price)
                                && momentums.AnnualizedSlope > 0
                           where x.DollarVolume >= DollarVolume &&
                               //x.Volume > 250000 &&
                               x.HasFundamentalData //&&
                               //x.Price > 20
                           //orderby x.DollarVolume descending
                           orderby momentums.AnnualizedSlope descending
                           select x;

            var topCoarse = filtered.Take(_universeSelectMaxStocks);

            return topCoarse.Select(x => x.Symbol);

        }

        //Select S&P500 stocks only:
        //Not possible at quantconnect so try to select common stock, primary share and dividend share
        public IEnumerable<Symbol> FineSelectionFunction(IEnumerable<FineFundamental> fine)
        {
            if (!_rebalanceFlag) return Enumerable.Empty<Symbol>();

            var filtered = from x in fine
                           where x.SecurityReference.SecurityType == "ST00000001" &&
                               x.SecurityReference.IsPrimaryShare                               
                           select x;

            // find piatroski score
            var piatroskiscore = from x in filtered
                                 let fs = FScore(
                                     x.FinancialStatements.IncomeStatement.NetIncome.TwelveMonths,
                                     x.FinancialStatements.CashFlowStatement.CashFlowFromContinuingOperatingActivities.TwelveMonths,
                                     x.OperationRatios.ROA.ThreeMonths, x.OperationRatios.ROA.OneYear,
                                     x.FinancialStatements.BalanceSheet.ShareIssued.ThreeMonths, x.FinancialStatements.BalanceSheet.ShareIssued.TwelveMonths,
                                     x.OperationRatios.GrossMargin.ThreeMonths, x.OperationRatios.GrossMargin.OneYear,
                                     x.OperationRatios.LongTermDebtEquityRatio.ThreeMonths, x.OperationRatios.LongTermDebtEquityRatio.OneYear, // fix need last year
                                     x.OperationRatios.CurrentRatio.ThreeMonths, x.OperationRatios.CurrentRatio.OneYear, // fix need last year 
                                     x.OperationRatios.AssetsTurnover.ThreeMonths, x.OperationRatios.AssetsTurnover.OneYear // fix need last year
                                 )
                                 where (fs >= 5)
                                 orderby fs descending
                                 select x;

            return piatroskiscore.Select(x => x.Symbol);

        }

        // this event fires whenever we have changes to our universe
        public override void OnSecuritiesChanged(SecurityChanges changes)
        {
            _securityChanges = changes;

            if (changes.AddedSecurities.Count > 0)
            {
                Log("Securities added: " + string.Join(",", changes.AddedSecurities.Select(x => x.Symbol.Value)));
            }
            if (changes.RemovedSecurities.Count > 0)
            {
                Log("Securities removed: " + string.Join(",", changes.RemovedSecurities.Select(x => x.Symbol.Value)));
            }
        }
        // Fire plotting events once per day:
        public override void OnEndOfDay()
        {
            ///Plotting
            //Assuming daily mode,dont chart in a smaller res and kill quota
            if (IsPlotting)
            {
                Plot(SP500IndexSymbol, "Price", _spyPriceClose);
                if (_isPlotSpyMovingAverage && _spyMovingAverage.IsReady)
                {
                    Plot(SP500IndexSymbol, _spyMovingAverage);
                }

                if (Portfolio.TotalPortfolioValue > 0)
                {
                    var accountLeverage = Portfolio.TotalAbsoluteHoldingsCost / Portfolio.TotalPortfolioValue;
                    Plot("Leverage", "Leverage", accountLeverage);
                }

                //Plot("STT", "Price", _sttPriceClose);
                //Plot("STT-M", "Momentum", _sttAnnualizedExponentialSlopeIndicator.Current.Value);
            }
        }
    }
}

using MathNet.Numerics;
using QuantConnect.Indicators;
using System;
using System.Linq;

//Copied from this forum:
//href https://www.quantconnect.com/forum/discussion/695/adjusted-slope--exponential-slope----annualized-slope--r-squuared--adjusted-slope/p1

namespace QuantConnect.Algorithm.CSharp.Helpers
{
    public class AnnualizedExponentialSlopeIndicator : WindowIndicator<IndicatorDataPoint>
    {
        public AnnualizedExponentialSlopeIndicator(int period)
            : base("AESI(" + period + ")", period)
        {
        }

        public AnnualizedExponentialSlopeIndicator(string name, int period)
            : base(name, period)
        {
        }

        protected override decimal ComputeNextValue(IReadOnlyWindow<IndicatorDataPoint> window, IndicatorDataPoint input)
        {
            if (window.Count < 3) return 0m;

            var xVals = new double[window.Count];
            var yVals = new double[window.Count];

            // load input data for regression
            for (int i = 0; i < window.Count; i++)
            {
                xVals[i] = i;
                // we want the log of our y values
                yVals[i] = Math.Log((double)window[window.Count - i - 1].Value);
            }

            //http://numerics.mathdotnet.com/Regression.html

            // solves y=a + b*x via linear regression
            var fit = Fit.Line(xVals, yVals);
            var intercept = fit.Item1;
            var slope = fit.Item2;

            // compute rsquared
            var rsquared = GoodnessOfFit.RSquared(xVals.Select(x => intercept + slope * x), yVals);

            // anything this small can be viewed as flat
            if (double.IsNaN(slope) || Math.Abs(slope) < 1e-25) return 0m;

            // trading days per year for us equities
            const int dayCount = 252;

            // annualize dy/dt
            var annualSlope = ((Math.Pow(Math.Exp(slope), dayCount)) - 1) * 100;

            // scale with rsquared
            //annualSlope = annualSlope * Math.Pow(rsquared, 2);
            annualSlope = annualSlope * rsquared;

            if (annualSlope >= (double)decimal.MaxValue || annualSlope <= (double)decimal.MinValue)
            {
                annualSlope = -1000;
                //Debug("Negative slope due to arithmic overflow");
            }

            return Convert.ToDecimal(annualSlope);

        }
    }
}