from QuantConnect.Data.UniverseSelection import *
import pandas as pd
import numpy as np
from scipy import stats

import decimal as d
import random

class BasicTemplateAlgorithm(QCAlgorithm):
    
    def Initialize(self):
        self.Debug("%%%%%%%%%%%%% Initialize")
        self.my_universe = []
        
        # Set the initial cash, start, and end dates for backtesting
        self.mycash = 30000
        self.mycash = 5000
        self.cash_per_trade = 5000
        self.SetCash(self.mycash)
        self.SetStartDate(2013,1,1)
        #self.SetEndDate(2018,11,11)
        self.SetBrokerageModel(BrokerageName.InteractiveBrokersBrokerage, AccountType.Margin) # Or override account type
        
        # Subscribe to data for securities returned in my universe 
        self.UniverseSettings.Resolution = Resolution.Minute
        self.UniverseSettings.MinimumTimeInUniverse = 0 
        self.AddUniverse(self.universe_filter_course, self.universe_filter_fine)
        
        # Schedule the 'trade' method to run
        # Add SPY to our data subscription so we can schedule from it (use defaults)
        
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.At(9, 31),
                         Action(self.my_trade))
                         
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.At(15, 50),
                         Action(self.liquidate_portfolio))
        
        self.Schedule.On(self.DateRules.EveryDay(),
                         self.TimeRules.At(11, 00),
                         Action(self.cancel_orders))
                         
        overlayPlot = Chart("Overlay Plot")
        self.AddChart(overlayPlot)
        
        #self.Schedule.On(
        #    self.DateRules.EveryDay('SPY'), 
        #    self.TimeRules.AfterMarketOpen('SPY', 1),       
        #    Action(self.my_trade))
    
    def universe_filter_fine(self, fine_data):
        columns = ['SecurityType', 'ShareClassStatus', 'ExchangeId', 'IsDepositaryReceipt', 'IsPrimaryShare']
        
        data_df = pd.DataFrame.from_records(
                [[getattr(s.SecurityReference, name) for name in columns] for s in fine_data],
                #[[s.ValuationRatios.PERatio, s.ValuationRatios.SalesYield] for s in fine_data],
                index=[s.Symbol for s in fine_data],
                columns=columns,)
        data_df = data_df.query("(SecurityType == 'ST00000001') and (IsDepositaryReceipt == False) and (IsPrimaryShare)")
        
        #data_df = data_df.sort_values(data_df.columns[0], ascending = True)
        data_df.sort_index(inplace=True)
        #data_df = data_df[:10]
        #self.Debug(data_df)
        
        #self.Debug("data is: {}".format(data_df.EVToEBITDA.values))
        self.universe = data_df.index.tolist()
        self.my_universe = data_df.index.tolist()
        
        #self.Debug("fine size is: {}".format(len(self.universe)))
        return self.universe

    def universe_filter_course(self, coarse_data):
        # With data_df already initialized globally, this should now just
        #   populate securities for values matching query instead.
        columns = ['Price', 'DollarVolume', 'HasFundamentalData', 'Volume']
        
        data_df = pd.DataFrame.from_records(
                [[getattr(s, name) for name in columns] for s in coarse_data],
                index=[s.Symbol for s in coarse_data],
                columns=columns,
                coerce_float=True)
                
        data_df = data_df.sort_values(by='Volume', ascending=False)

        data_df = data_df.query("HasFundamentalData " \
            "and (DollarVolume > @data_df.DollarVolume.quantile(.20))" \
            "and (DollarVolume < @data_df.DollarVolume.quantile(.21))")
        '''
        data_df = data_df.query("HasFundamentalData ")
        data_df = data_df.sort_values('DollarVolume')
        data_df = data_df[data_df.DollarVolume > 1000000]
        data_df = data_df[50:100]
        '''
        self.StocksDailyData =data_df
        if len(data_df) == 0:
            return
        self.my_universe = data_df.index.tolist()
        
        return data_df.index.tolist()

    def my_trade(self):
        # See how many securities we have in our Securities list
        '''
        for universe in self.UniverseManager.Keys:
            self.Debug("universe name: {}".
                format(universe))
        '''    
        
        self.Debug("%%%%%%%%%%%%% my_trade")
        for universe in self.UniverseManager.Values:
            self.Debug("universe count: {}".
                format(universe.Members.Count))
        
        if len(self.my_universe) == 0:
            self.Debug('Empty universe')
            return
        
        stock_list = []

        for stock in self.my_universe:
            stock_list.append(stock)
            stockID = self.AddEquity(stock.Value)
            
        h1 = self.History(stock_list, 31, Resolution.Daily)
        h2 = self.History(stock_list, 1, Resolution.Minute)
        #self.Debug("\nstock_list - 1: %s "%([s.Value for s in stock_list]))
        #self.Debug("\nstock_list - 2: %s "%([s.Value for s in self.my_universe]))
        #self.Debug("\nstock_list - 3: %s "%([s[0] for s in h2.index.tolist()]))
        
        if not 'close' in h1.keys() or \
            not 'open' in h2.keys():
            self.Debug('Missing Data')
            return
        
        noMatch = returnNotMatches(h1.index.levels[0],h2.index.levels[0])
        
        h1.drop(noMatch[0], level='symbol', inplace=True)
        h2.drop(noMatch[1], level='symbol', inplace=True)
        
        #h1.index.get_level_values(0).unique()
        
        closes = h1['close'].unstack(level=0).reset_index(drop=True).values
        opens = h1['open'].unstack(level=0).append(
                h2['open'].unstack(level=0), 
                ignore_index=True)[1:].reset_index(drop=True).values

        gap = opens - closes

        gap_z = stats.zscore(gap, axis=0, ddof=1)

        min_gap_z = np.nanmin(gap_z[-1,:])
        max_gap_z = np.nanmax(gap_z[-1,:])
        
        k_min = np.nanargmin(gap_z[-1,:])
        k_max = np.nanargmax(gap_z[-1,:])

        #self.Debug("Min (%s) %f Max (%s) %f"%(self.my_universe[k_min], min_gap_z, self.my_universe[k_max], max_gap_z))
            
        i=0
        buyList = []
        sellList = []
        self.Debug("NOMBER IN UNIVERSe {}".format(len(self.my_universe)))

        #if len(stock_list) != len(h1.index.levels[0]) or \
        #    len(stock_list) != len(h2.index.levels[0]):
        #if len(stock_list) != len(h1.index.get_level_values(0).unique()) or \
        #    len(stock_list) != len(h2.index.get_level_values(0).unique()):
        #        
        #    self.Debug('===================')
        #    self.Debug(len(stock_list))
        #    self.Debug("\nstock_list: %s "%([s.Value for s in stock_list]))
        #    self.Debug(len(h1.index.get_level_values(0).unique()))
        #    self.Debug("\nh1.index.levels[0]: %s "%([s for s in h1.index.get_level_values(0).unique()]))
        #    self.Debug(len(h2.index.get_level_values(0).unique()))
        #    self.Debug("\nh2.index.levels[0]: %s "%([s for s in h2.index.get_level_values(0).unique()]))
            
            
        #for sid in h1.index.levels[0]:
        '''
        allStock = h1.index.get_level_values(0).unique()
        maxZ = gap_z[-1,]/min_gap_z
        maxZval = np.nanmax(maxZ)
        if maxZval >= 0.75:
            maxZpos = np.nanargmax(maxZ)
            buyList.append(allStock[maxZpos])
        '''
        self.Debug(gap_z[-1])
        for sid in h1.index.get_level_values(0).unique():
            if gap_z[-1,i] / min_gap_z >= 0.999:
            #if gap_z[-1,i] / min_gap_z >= 0.75:    
                self.Debug("->buy: %s  :: %f"%(sid,gap_z[-1,i]))
                buyList.append(sid)
            elif gap_z[-1,i] / max_gap_z >= 0.95:
                self.Debug("->sell: %s  :: %f"%(sid,gap_z[-1,i]))
                sellList.append(sid)
            i += 1
        #buyPower = (self.Portfolio.Cash * leverage) / (len(buyList) + len(sellList))
        #self.Debug('VALUES !!!!!!!!!!!!!!!!!!!!!!!!!!')
        #self.Debug(self.Portfolio.Cash)
        #self.Debug(self.Portfolio.TotalPortfolioValue)
        #self.Debug(self.mycash)
        #buyPower = ((self.mycash * 1) / (len(buyList)))-100
        
        percent = 1. / (len(buyList))
        
        self.Plot("Overlay Plot", "Num Stocks Buy", len(buyList))
        
        #buyPower = float(max(self.cash_per_trade,buyPower/5))
        #buyPower = float(min(self.cash_per_trade,self.portfolio.cash/5))
        
        self.Debug("\nBUYS: %s \nSELLS: %s"%([s for s in buyList],[s for s in sellList]))
        
        self.Log('Cash: ' + str(self.Portfolio.Cash))
        self.Log('TotalFees: ' + str(self.Portfolio.TotalFees))

        self.Debug('Checking Holdings ...')
        for kvp in self.Portfolio:
            holdings = kvp.Value
            symbol = holdings.Symbol
            
            if (holdings.Invested):
                self.Debug("Symbol: {} -> Price: {}".format(symbol, holdings.AveragePrice))
                
        for sid in buyList:
            #stock = self.AddEquity(sid.Value).Symbol
            price = float(self.Securities[sid].Price)
            if price == 0: continue
        
            #self.Debug(self.StocksDailyData.head(50))
            
            self.Debug(h1.loc[sid]['close'][0])
            self.Debug(h1.loc[sid]['volume'][0])
            self.Debug(h1.loc[sid]['close'][0]*h1.loc[sid]['volume'][0])
            prevDayDV = h1.loc[sid]['close'][0]*h1.loc[sid]['volume'][0]
            self.Debug('//////////////////////////////////////////////////////////')
            
            self.Debug(prevDayDV)
            
            #allocatedValue = float(min(prevDayDV/10, self.Portfolio.TotalPortfolioValue/(len(buyList))))
            allocatedValue = float(self.Portfolio.TotalPortfolioValue/(len(buyList)))
            shares = allocatedValue / price
            
            #if allocatedValue < self.cash_per_trade:
            #    continue
            
            
            #shares = (prevDayDV/10) / price
            
            #shares = self.cash_per_trade/price
            
            #self.Debug(self.Portfolio.GetBuyingPower(sid, OrderDirection.Buy))
            #shares = self.CalculateOrderQuantity(sid, percent)
            if shares > 0:
                self.Buy(sid, shares)
                #self.MarketOrder(sid, shares)
                #self.LimitOrder(sid, shares, price)
                #self.SetHoldings(sid.Value, shares)
                self.Debug("Buy Stock : {} , # Shares {} , price {}".format(sid, shares, price))
                #self.MarketOrder(sid, shares)
                #self.SetHoldings(sid, percent)
                #self.StopLimitOrder(sid, shares, decimal.Decimal(price  * 0.90), decimal.Decimal(price  * 0.90))
                #self.StopMarketOrder(sid, shares, decimal.Decimal(price  * 0.90));
                #self.LimitOrder(sid.Value, shares)
                #order(sid,shares,limit_price=price,stop_price=price)

        '''
        for sid in sellList:
            #stock = self.AddEquity(sid.Value).Symbol
            price = float(self.Securities[sid].Price)
            if price == 0: continue
            shares = buyPower/price
            if shares > 0:
                self.Debug("Stock : {} , # Shares {} , price {}".format(sid, shares, price))
               #self.MarketOrder(sid, -shares)
                #self.StopLimitOrder(sid, shares, decimal.Decimal(price  * 1.10), decimal.Decimal(price  * 1.10))
                #self.StopMarketOrder(sid, shares, decimal.Decimal(price  * 1.10));
                
                
                #self.Debug("Stock : {} , # Shares {} , price {}".format(sid, shares, price))
                #self.LimitOrder(sid.Value, shares)
                #order(sid,-shares,limit_price=price,stop_price=price)
            
        '''
        '''
        #for security in self.my_universe:
        #    self.Debug('#####################')
        #    self.SetHoldings(security, weight)
        '''
        
    def liquidate_portfolio(self):
        
        self.Debug('Checking Holdings 2222222222222...')
        for kvp in self.Portfolio:
            holdings = kvp.Value
            symbol = holdings.Symbol
            
            if (holdings.Invested):
                self.Debug("Symbol: {} -> Price: {}".format(symbol, holdings.AveragePrice))
                
                
                
        for symbol in self.Portfolio.Keys:
            self.SetHoldings(symbol, 0)

      
    def cancel_orders(self):
        self.Debug('=====================================================================')
        openOrders = self.Transactions.GetOpenOrders()
        if len(openOrders)> 0:
            for x in openOrders:
                 self.Transactions.CancelOrder(x.Id)
################https://www.quantconnect.com/docs/algorithm-reference/overview

def returnNotMatches(a, b):
    return [[x for x in a if x not in b], [x for x in b if x not in a]]
    
def to_dataframe(data_c, properties, labels):
    # Helper function to make a dataframe from the coarse object. 
    # Then we can use all the handy dataframe methods.
    # Set the 'coerce_float' parameter to get meaningful datatypes
    # otherwise all the fields will be un-useful generic objects.

    data = [[getattr(stock, name) for name in properties] for stock in data_c]
    symbols = [stock.Symbol for stock in data_c ]
    
    data_df = pd.DataFrame.from_records(
            data, 
            index=symbols, 
            columns=labels, 
            coerce_float=True)
            
    return data_df