/*
 * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
 * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
*/

using System.Collections.Generic;
using System.Linq;
using QuantConnect.Data.Fundamental;
using QuantConnect.Data.Market;
using QuantConnect.Data.UniverseSelection;

namespace QuantConnect.Algorithm.CSharp
{
    /// <summary>
    /// Demonstration how to group stocks by Morningstar Asset Classification and then compare fundamental factors to select the best stock.
    /// This was inspired by the Fundamental Factor Long Short Strategy:
    /// - https://www.quantconnect.com/tutorials/strategy-library/fundamental-factor-long-short-strategy
    /// </summary>
    /// <meta name="tag" content="using data" />
    /// <meta name="tag" content="universes" />
    /// <meta name="tag" content="coarse universes" />
    /// <meta name="tag" content="fine universes" />
    public class FundamentalFactorLongUniverse : QCAlgorithm
    {
        private const int _numberOfSymbolsCoarse = 1000;
        private const int _numberOfSymbolsFine = 2;
        
        // initialize our changes to nothing
        private SecurityChanges _changes = SecurityChanges.None;

        private readonly Dictionary<Symbol, decimal> _dollarVolumeBySymbol = new Dictionary<Symbol, decimal>();

        public override void Initialize()
        {
            UniverseSettings.Resolution = Resolution.Daily;
			SetSecurityInitializer(s => s.SetFeeModel(new ConstantFeeModel(.005m)));
            SetStartDate(2010, 01, 01);
            SetEndDate(2012, 01, 01);
            SetCash(50000);

            // This add universe method accepts two parameters:
            // - coarse selection function: accepts an IEnumerable<CoarseFundamental> and returns an IEnumerable<Symbol>
            // - fine selection function: accepts an IEnumerable<FineFundamental> and returns an IEnumerable<Symbol>
            AddUniverse(CoarseSelectionFunction, FineSelectionFunction);
        }

        // sort the data by daily dollar volume and take the top 'NumberOfSymbolsCoarse'
        public IEnumerable<Symbol> CoarseSelectionFunction(IEnumerable<CoarseFundamental> coarse)
        {
            

            // Coarse Filtering:
            // - the stocks must have fundamental data 
            // - volume must be greater than 0
            // - price must be greater than 0
            // - avoids Nasdaq test symbols
            // - ordered by DollarVolume descending
            var sortedByDollarVolume =
                (from x in coarse
                     where x.Symbol != "ZWZZT" ||   // Nasdaq test symbol
                            x.Symbol != "ZXZZT" ||  // Nasdaq test symbol
                            x.Symbol != "ZVZZT"     // Nasdaq test symbol
                 where x.HasFundamentalData && x.Volume > 0 && x.Price > 0
                 orderby x.DollarVolume descending
                 select x).Take(_numberOfSymbolsCoarse).ToList();

            // Add coarse symbols to the dollarVolumeBySymbol dictionary for use later in the fine selctor
            _dollarVolumeBySymbol.Clear();
            foreach (var i in sortedByDollarVolume)
            {
                _dollarVolumeBySymbol[i.Symbol] = i.DollarVolume;
            }
            return _dollarVolumeBySymbol.Keys;
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="fine"></param>
        /// <returns></returns>
        public IEnumerable<Symbol> FineSelectionFunction(IEnumerable<FineFundamental> fine)
        {

            // Split stocks into groups by Asset Classification - MorningstarIndustryCode)
            // - splitting the stocks up by 148 classifications alows us to compare stocks in the same industry (apples to apples)
            // - only selects US stocks
            // - only selects stocks from the New York Stock Exchange or Nasdaq
            // - will not select stocks within 180 days of IPO date
            // - only selects Grade A growth grade stocks
            // - order by dollar volume from the dollarVolumeBySector dictionary from the coarse selector 
            Dictionary<int, IOrderedEnumerable<FineFundamental>> topFineBySector =
                (from x in fine
                     // Group by MorningstarIndustryCode
                 group x by x.AssetClassification.MorningstarIndustryCode into g
                 let y = from item in g
                         where item.CompanyReference.CountryId == "USA" &&
                        (item.CompanyReference.PrimaryExchangeID == "NYS" || item.CompanyReference.PrimaryExchangeID == "NAS") &&
                         (Time - item.SecurityReference.IPODate).Days > 180 &&
                         item.AssetClassification.GrowthGrade == "A"
                         orderby _dollarVolumeBySymbol[item.Symbol] descending
                         select item
                 select new { g.Key, Value = y }
                 ).ToDictionary(x => x.Key, x => x.Value);

            // Collections to order sectors by different valuations
            Dictionary<int, List<FineFundamental>> sectorByBookValuePerShare = new Dictionary<int, List<FineFundamental>>();
            Dictionary<int, List<FineFundamental>> sectorByOperationMargin = new Dictionary<int, List<FineFundamental>>();
            Dictionary<int, List<FineFundamental>> sectorByEvToEbit = new Dictionary<int, List<FineFundamental>>();

            // Order each sector by 3 valuations
            foreach (var sector in topFineBySector)
            {
                var sectorNumber = sector.Key;
                var sectorFine = sector.Value;

                // Order each sector by book value per share ratio
                var sectorFilteredByBookValuePerShare = sectorFine.Select(x => x)
                    .OrderBy(y => y.ValuationRatios.BookValuePerShare)
                    .Select(f => f).ToList();                

                // Order each sector by OperationMargin
                var sectorFilteredByOperationMargin = sectorFine.Select(x => x)
                    .OrderByDescending(z => z.OperationRatios.OperationMargin.OneYear)
                    .Select(f => f).ToList();               

                // Order each sector by EvToEbit ratio.
                var sectorFilteredByEvToEbit = sectorFine.Select(x => x)
                    .OrderBy(r => r.ValuationRatios.EVtoEBIT)
                    .Select(f => f).ToList();

                // Add ordered secotrs to the sector valuation collections
                sectorByBookValuePerShare.Add(sectorNumber, sectorFilteredByBookValuePerShare);
                sectorByOperationMargin.Add(sectorNumber, sectorFilteredByOperationMargin);
                sectorByEvToEbit.Add(sectorNumber, sectorFilteredByEvToEbit);

            }

            // Put all the sectors in the collections in list's so we can use the index to calculate a score
            var bookValuePershareList = sectorByBookValuePerShare.SelectMany(x => x.Value)
                .OrderBy(y => y.ValuationRatios.BookValuePerShare)
                .Select(x => x.Symbol).Take(25).ToList();

            var OperationMarginList = sectorByOperationMargin.SelectMany(x => x.Value)
                .OrderByDescending(z => z.OperationRatios.OperationMargin.OneYear)
                .Select(x => x.Symbol).Take(25).ToList();

            var EvToEbitList = sectorByEvToEbit.SelectMany(x => x.Value)
                .OrderBy(r => r.ValuationRatios.EVtoEBIT)
                .Select(x => x.Symbol).Take(25).ToList();

            Dictionary<Symbol, double> symbolDic = new Dictionary<Symbol, double>();

            foreach (var symbol in bookValuePershareList)
            {
                // Get the index of each symbol in each valuation list. We'll use this value multiplied by a multiplier to get a score.
                var value = bookValuePershareList.FindIndex(a => a == symbol);
                var quality = EvToEbitList.FindIndex(a => a == symbol);
                var momentum = OperationMarginList.FindIndex(a => a == symbol);

                // Multiply each valuation by a weight:
                // - Value: Multiplied by 40% (.4)
                // - Quality: Multiplied by 40% (.4)
                // - Momentum: Multiplied by 20% (.2)
                var finalValue = (value * .4) + (quality * .4) + (momentum * .2);

                symbolDic.Add(symbol, finalValue);
            }

            // Return the symbol (key) ordered by score (value) ascending and take the defined number of symbols fine
            return symbolDic.Select(x => x)
                .OrderBy(x => x.Value)
                .Select(x => x.Key).Take(_numberOfSymbolsFine);
        }

        //Data Event Handler: New data arrives here. "TradeBars" type is a dictionary of strings so you can access it by symbol.
        public void OnData(TradeBars data)
        {
            // if we have no changes, do nothing
            if (_changes == SecurityChanges.None) return;

            // liquidate removed securities
            foreach (var security in _changes.RemovedSecurities)
            {
                if (security.Invested)
                {
                    Liquidate(security.Symbol);
                    Debug("Liquidated Stock: " + security.Symbol.Value);
                }
            }

            // we want 20% allocation in each security in our universe
            foreach (var security in _changes.AddedSecurities)
            {
                SetHoldings(security.Symbol, 0.5m);
                Debug("Purchased Stock: " + security.Symbol.Value);
            }

            _changes = SecurityChanges.None;
        }

        // this event fires whenever we have changes to our universe
        public override void OnSecuritiesChanged(SecurityChanges changes)
        {
            _changes = changes;

            if (changes.AddedSecurities.Count > 0)
            {
                Debug("Securities added: " + string.Join(",", changes.AddedSecurities.Select(x => x.Symbol.Value)));
            }
            if (changes.RemovedSecurities.Count > 0)
            {
                Debug("Securities removed: " + string.Join(",", changes.RemovedSecurities.Select(x => x.Symbol.Value)));
            }
        }
    }
}