### <summary>
### Basic template algorithm simply initializes the date range and cash. This is a skeleton
### framework you can use for designing an algorithm.
### </summary>
import numpy as np
import pandas as pd
import scipy
import sys
from datetime import datetime, timedelta

from clr import AddReference
AddReference("System")
AddReference("QuantConnect.Algorithm")

AddReference("QuantConnect.Indicators")
AddReference("QuantConnect.Common")
import statsmodels.api as sm
from QuantConnect import *
from QuantConnect.Algorithm import *
from QuantConnect.Data import *
from QuantConnect.Indicators import *
from QuantConnect.Orders import *
from QuantConnect.Securities import *
from System.Collections.Generic import List
from scipy.optimize import minimize
#import matplotlib.pyplot as plt # only works in Notebook
import decimal
from System.Collections import *
RESEARCHMODE= False # set this to False to toggle for simulated or live trading
VERBOSEMODE= True  # set thi
VERBOSEMODE2= True # set thi


class BasicTemplateAlgorithm(QCAlgorithm):
    
    def Initialize(self):
        # Set the cash we'd like to use for our backtest
        # This is ignored in live trading 
        self.SetCash(100000)
        self.excess_liquidity = decimal.Decimal(50000.00)
        # Start and end dates for the backtest.
        # These are ignored in live trading.
        self.SetStartDate(2017,11,05)
        self.SetEndDate(2017,11,12)
        self.benchmark = self.AddEquity("SPY",Resolution.Minute).Symbol #used for references to scheduling  
        strbench = self.benchmark.ToString() # must be in string format
        self.bullish_stock =  self.AddEquity("TQQQ", Resolution.Minute).Symbol 
        self.bearish_stock =  self.AddEquity("TMF", Resolution.Minute).Symbol 
        self.small_cap_stock =  self.AddEquity("TNA", Resolution.Minute).Symbol 
        self.mid_cap_stock =  self.AddEquity("IJH", Resolution.Minute).Symbol 
        self.vxx =  self.AddEquity("VXX", Resolution.Minute).Symbol 
        self.xiv=  self.AddEquity("XIV", Resolution.Minute).Symbol 
        self.spy =  self.AddEquity("TQQQ", Resolution.Minute).Symbol 
        self.shortSpy =  self.AddEquity("TLT", Resolution.Minute).Symbol 

        self.stocks = [self.bullish_stock,
                          self.bearish_stock,
                          self.small_cap_stock,
                          self.mid_cap_stock,
                          self.xiv]        

        self.skippedSecurities =  [
            self.AddEquity("AGG", Resolution.Minute).Symbol,
            self.AddEquity("TIPS", Resolution.Minute).Symbol 
            ]
            
        self.permanentStocks=  [
            self.AddEquity("AGG", Resolution.Minute).Symbol,
            self.AddEquity("TIPS", Resolution.Minute).Symbol,
            self.AddEquity("TLT", Resolution.Minute).Symbol,
            self.AddEquity("SDY", Resolution.Minute).Symbol            
            ]     
        self.boughtPermanent = False   
        #set up variables to calculate margin
        self.day= datetime.today().day  
        self.timestep=0  
        self.margin_req=0  
        self.totalShorts=0  
        self.totalLongs=0  
        self.cash=0  
        self.weightsp = { self.permanentStocks[0]: 0.50 , self.permanentStocks[1]: 0.10,self.permanentStocks[2]: 0.20,self.permanentStocks[3]: 0.20,}    
        self.track_orders = 1    # toggle on|off
        self.n = 0
        self.s = np.zeros_like(self.stocks)
        self.x0 = np.zeros_like(self.stocks)
        self.x1 = 1.0*np.ones_like(self.stocks)/len(self.stocks)
        self.eps = 0.01
        self.tol = 1.0e-6    #assume convergence is 10 time SLSQP ftol of 1e-6
        self.valid_constraint_count = 0
        self.opt_pass_count = 0
        self.run_count = 0
        self.eps_vals = []
        self.portvalue =  self.Portfolio.get_TotalPortfolioValue()     
        self.RISK_LEVEL = 1.85
        self.is_first_of_week = False   
        self.is_trading_day = False   
        self.rebalance_threshhold = 500
    
        
        self.OTHER_MIX = 0.10  # place this as percentage of non-permanent portfolio that you WANT invested in this strategy. 
                                  # this is percentage of  1-self.permanentRatio. 1.00 = 100% into aggressive strategy
        self.permanentRatio= 0.90 # store percentage of portfolio dedicated to permanent positioins and excluded from this algo
        self.waits_max = 3    # trading days    
        self.profit_threshhold = 0.3 # percentage terms 1.0 = 1 pct
        self.enableProfitTaking = True
        self.account_leverage = self.Portfolio.TotalAbsoluteHoldingsCost / self.Portfolio.TotalPortfolioValue
        self.mx_lvrg = 0 
        # stop losses according to rising intraday threshhold
        self.loss_threshhold =  20.00  # percentage terms 1.0 = 1 pct
        self.enableLossLimiting= False      
        # makes sure at least one position is short at all times. (for quantopian contest rules)
        self.alwaysShort = True
        self.short_stock  =  self.AddEquity("SQQQ", Resolution.Minute)      
        self.maxprice = {}    # Keys are symbols, values are lists of prices.
        for s in self.stocks:
             self.maxprice[s] = 00   
        if self.account_leverage > self.mx_lvrg:  
            self.mx_lvrg = self.account_leverage  
        self.current_holdings = [] # used to manage portfolio
        self.waits = {} # used to delay repetitive actions
        self.total_trades =0
        self.sliceTime = datetime.today()



        try:
            self.Schedule.On(self.DateRules.EveryDay(),  self.TimeRules.At(10, 0), Action(self.buyPermanent))

            self.Schedule.On(self.DateRules.MonthStart(strbench),self.TimeRules.AfterMarketOpen(strbench,20), Action(self.refreshPermanent))            
                
            
        except:    
            self.Log('An unknown error occurred trying to Initialize.' +  str(sys.exc_info()[0]) )
     
        
        
    def addStock(self,s,pct):
        # sometimes we mistakenly call the stock by its object, not by its Symbol property because of borrowed code
        # this catches that and adjusts for it.
        if VERBOSEMODE: self.Log('line 192 addStock reached') 

        try:
            if hasattr(s,'Symbol'):
                sym = s.Symbol
            else:
                #make sure we are an equity already
                if not isinstance(s,Securities.Equity.Equity):
                    sym = self.AddEquity(s,Resolution.Minute).Symbol
                else:
                    sym = s
            self.SetHoldings(sym,pct)
            return True
        except:
            if VERBOSEMODE: self.Log('An unknown error occurred trying to optimize the portfolio within allocate.', sys.exc_info()[0])   
            return False    

    
    def OnData(self,slice):
        try:
            self.data = slice

            if (False in [slice.ContainsKey(x) for x in self.stocks]) \
                and (False in [slice.ContainsKey(x) for x in self.permanentStocks]) \
                and (False in [slice.ContainsKey(x) for x in self.skippedSecurities]) :
                    
                    if VERBOSEMODE: self.Log('exiting OnData no valid stocks in current data stream')                      
                    return  
            self.sliceTime = slice.Time    
            if self.sliceTime.weekday()<> 1 : #tuesday is day 1
                self.is_trading_day = False
            if self.sliceTime.weekday()<> 0 : # monday
                self.is_first_of_week = False 
            #if VERBOSEMODE:self.Log("line number 193 is_trading_day {} is_first_of_week {} weekday {} n date {} {}:{} ".format( self.is_trading_day, self.is_first_of_week,self.sliceTime.weekday(), self.sliceTime.day,self.sliceTime.hour,self.sliceTime.minute))

        except:
            self.Log('An unknown error occurred in OnData'), sys.exc_info()[0]   
            return    
 
           
    def refreshPermanent(self):
        self.boughtPermanent = False
        self.buyPermanent
        
    def buyPermanent(self): 
        try:
            if not self.boughtPermanent:
                for stock in self.permanentStocks:
                    sym = self.symbolAsString(stock)
                    if VERBOSEMODE2:self.Log("line number 198 stock {}".format(stock))
                    if self.Securities[sym].Holdings :
                        if VERBOSEMODE2:self.Log('permanentRatio symbol {} permanentRatio is  {} weightsp[stock]  is {} ' .format(sym,self.permanentRatio, self.weightsp[stock]))
                        adjustedWeight= self.permanentRatio * self.weightsp[stock]
                        if VERBOSEMODE2:self.Log('adjustedWeight is {}' .format(adjustedWeight))
                        self.place_order(sym,adjustedWeight,False)
                self.boughtPermanent= True        
        except:
            self.Log('An unknown error occurred in buyPermanent function.' +  str(sys.exc_info()[0]) )
            return False                 
   
                
    def set_is_first_of_week(self):
        #if datetime.today().weekday() == 0: # Monday
        self.is_first_of_week = True
        self.is_trading_day  = False
        #else:
           #self.is_first_of_week = False  
        if VERBOSEMODE:self.Log("line number 232 is_first_of_week  date {} {}:{} ".format( self.sliceTime.day,self.sliceTime.hour,self.sliceTime.minute))
          
    def set_is_trading_day(self):
        self.is_trading_day = True
        self.is_first_of_week = False        
        if VERBOSEMODE:self.Log("line number 307 is_trading_day  date {} {}:{} ".format( self.sliceTime.day,self.sliceTime.hour,self.sliceTime.minute))
           
        
    def record_leverage(self):
        print("Total leverage: %d" % self.account_leverage)




    def place_order(self,  stock, percent,liquidateExisting=False, resolution=None):
        try:     
            if VERBOSEMODE: self.Log('line 309 placeOrder reached')
            if  resolution is None:
                resolution = Resolution.Minute
            if VERBOSEMODE: self.Log("reached line 313. liquidateExisting now set to {}". format(liquidateExisting) )
            sym = self.symbolAsString(stock)    
               
            shares = self.get_desired_allocation( sym, percent)
            current_shares = self.Portfolio[sym].Quantity
            if VERBOSEMODE: self.Log(" line 319 found shares for symbol {} of {}". format(sym,current_shares))
            needShares = shares != current_shares
            if VERBOSEMODE: self.Log("reached line 321 need shares? {}".format(needShares))

            if shares != current_shares:
                #  allowing predetermined period to first elapse
                if  self.waits.has_key(stock) :
                    if self.waits[stock] > 0:
                        if VERBOSEMODE: self.Log("Waiting for stock  to wake up from hibernation  {}".format(stock) )
                        return False
                    else:
                        self.total_trades += 1  

            
            if isinstance(percent,decimal.Decimal) :
                pct=round(percent,4)
            else:
                pct = round(decimal.Decimal(percent),4)
            if VERBOSEMODE2: self.Log(" line 264 allocating symbol {} pct used {} for fraction {} ". format(sym,pct,percent))

            if not self.Securities[sym].Holdings :
                if VERBOSEMODE: self.Log('line 265  not found {} in holdings. Adding equity ' .format(sym)) 
                self.AddEquity(sym,resolution)
 
            if VERBOSEMODE2: self.Log('line 272 would set symbol  {} to percent  {}' .format(sym,pct)) 
            self.SetHoldings(sym,pct,liquidateExisting)
            amt = self.Portfolio[sym].Quantity
            if VERBOSEMODE2: self.Log('line 273 successfully set holding for symbol {} pct {} at {}:{} ' .format(sym,pct,self.sliceTime.hour,self.sliceTime.minute)) 
            if VERBOSEMODE2: self.Log('line 276 immediately after quantity is  {}' .format(amt)) 
           
            return True
        except:
            self.Log('An unknown error occurred in place_order function for stock {} error:{}' .format(sym, sys.exc_info()[0]) )
            return False             
        
        
    def get_desired_allocation(self,  stock, percent):
        if VERBOSEMODE: self.Log('line 358 getting needed shares for symbol {} percent {}' .format(stock,percent))         
        sym= self.symbolAsString(stock)
        pct = decimal.Decimal(percent)
        current_price =self.Securities[sym].Price
        if current_price == 0: #avoid division by 0
            if VERBOSEMODE: self.Log('line 291 current_price for symbol {} is {}' .format(sym,current_price))
            return 0
        shares = int((self.Portfolio.TotalPortfolioValue * pct)/current_price)
        if VERBOSEMODE: self.Log('line 369 needed shares for symbol {} are {}' .format(sym,shares)) 
       
        return shares
     
    def get_holding_size(self, stock):
        try:
            if VERBOSEMODE: self.Log('line 333 get_holding_size reached') 
            sym = self.symbolAsString(stock)
                    
            return ( self.Portfolio[sym].Quantity * self.Portfolio[sym].AveragePrice) / self.Portfolio.TotalPortfolioValue
        except:       
            self.Log('An unknown error occurred reading results of get_holding_size.{}'.format( sys.exc_info()[0]))   
            return 
            
    def allocate(self):   
        if VERBOSEMODE: self.Log('line 340 allocate reached at date{} hour {} minute {}'.format(self.sliceTime.day,self.sliceTime.hour,self.sliceTime.minute)) 
        try:        
            if not hasattr(self,"run_count"):
                self.run_count = 1
            else:
                self.run_count += 1
                
    
            if RESEARCHMODE:
                period = 17 # * 390
                prices = self.get_history_from_list(self.stocks,period,Resolution.Minute, 'close').dropna()#.bfill().ffill()
            else:
                period = 17 * 390
                prices = self.get_history_from_list(self.stocks,period,Resolution.Minute, 'close').dropna()#.bfill().ffill()
                if VERBOSEMODE: self.Log('line  354 in allocate prices are {}'. format(prices)) 
    
            ret = prices.pct_change()[1:].as_matrix()
            if VERBOSEMODE: self.Log("price pct change as matrix is {} line 395". format( ret))
    
            ret_mean = prices.pct_change().mean()
            ret_std = prices.pct_change().std()
            ret_norm = ret_mean/ret_std
            ret_norm = ret_norm.as_matrix()
          
    
            ret_norm_max = np.max(ret_norm)
            eps_factor = 0.9 if ret_norm_max >0 else 1.0
            self.eps = eps_factor*ret_norm_max        

            bnds = []
            limits = [0,1]

            for stock in self.stocks:
                bnds.append(limits)
    
            bnds = tuple(tuple(x) for x in bnds)
            if VERBOSEMODE: self.Log("line 419 bnds is {} ". format( bnds))
   
            cons = ({'type': 'eq', 'fun': lambda x:  np.sum(x)-1.0},
                    {'type': 'ineq', 'fun': lambda x:  np.dot(x,ret_norm)-self.eps})
    
            if VERBOSEMODE: self.Log("line 22 cons is {} ". format(cons))

            if VERBOSEMODE: self.Log("before minimize line 428 self.variance {}| self.x1 {} | self.jac_variance {} is " .format(self.variance,self.x1,self.jac_variance))

            res = minimize(self.variance, self.x1, args=ret,jac=self.jac_variance, method='SLSQP', bounds=bnds, constraints=cons)
            if VERBOSEMODE: self.Log("res is {} " .format( res))
            
        except:       
            if VERBOSEMODE: self.Log('An unknown error occurred trying to optimize the portfolio within allocate.', sys.exc_info()[0])   
            return  
        try:
            allocation = np.copy(self.x0)    
            if res.success:    # if SLSQP declares success
                if self.opt_pass_count is None:
                    self.opt_pass_count =1
                else:
                    self.opt_pass_count += 1

                weighted_ret_norm = np.dot(res.x,ret_norm)

                w_ret_constraint = weighted_ret_norm - self.eps + self.tol
               
                if VERBOSEMODE: self.Log("w_ret_constraint is {} ". format( w_ret_constraint))
                if(w_ret_constraint > 0): # and constraint is actually met
                    self.valid_constraint_count += 1
                    allocation = res.x
                    allocation[allocation<0] = 0
                    denom = np.sum(allocation)
                    if denom > 0:
                        allocation = allocation/denom 

                    msg = "{0} runs, {1} SLSQP passes, {2} constraints passed".format(
                        self.run_count, self.opt_pass_count,
                        self.valid_constraint_count)
                    if(self.run_count>1000):self.Log(msg)
                else:

                    self.Log("constraint fail, SLSQP status = {0}".format(res.status))
                    pass
            else:

                self.Log("SLSQP fail, SLSQP status = {0}".format(res.status))
                pass        
            self.n += 1 # increment number of allocations
            self.s += allocation
            
        except:       
            self.Log('An unknown error occurred reading results of optimize.{}'.format( sys.exc_info()[0]))   
            return    
        if VERBOSEMODE: self.Log( "allocation is now {}".format(allocation))
        
        if self.alwaysShort:
            self.reallocateToShort
       
        if self.is_trading_day:
            if VERBOSEMODE:self.Log("line number 482 is_trading_day set to True on {}: {}".format(self.is_trading_day,self.sliceTime.hour,self.sliceTime.minute))
            self.trade()
        else:
            if VERBOSEMODE:self.Log("line number 485 set_is_trading_day set to False on {}: {}".format(self.sliceTime.hour,self.sliceTime.minute))
        
        if VERBOSEMODE:self.Log("line number 442 _completed allocate function  on {}: {}".format(self.sliceTime.hour,self.sliceTime.minute))

          
    def GetShiftingWindows(self,thelist, size):
        if VERBOSEMODE: self.Log('line 398 GetShiftingWindows reached') 

        return [ thelist[x:x+size] for x in range( len(thelist) - size + 1 ) ]


    def reallocateToShort(self):
        try:
            if VERBOSEMODE: self.Log('line 405 reallocateToShort reached') 

            if self.n > 0 : 
                allocation = self.s
                n=self.n
                if VERBOSEMODE: self.Log( 'allocation in reallocateToShort is \n{} of type {} '.format(allocation, type(allocation)))
                if VERBOSEMODE: self.Log( 'old allocation is \n{}'.format(n))
                #find the biggest allocation
                
                maxAl = allocation.max()
                if VERBOSEMODE: print 'maxAl is \n',maxAl               
                lastPos = len(self.stocks)-1
                doOnce = True
                for i,stock in enumerate(self.stocks):
                    if allocation[i]==maxAl:
                        doOnce = False # so this is run only one time.
                        # this is stock we want. we are going to allocate 10% of its allocation to short its reverse etf (2 x negative = positive)
                        # to fulfill contest short requirements
                        previousShort = self.short_stock
                        if stock == self.bullish_stock:
                            self.short_stock = self.bearish_stock
                        elif stock == self.bearish_stock:
                            self.short_stock  = self.bullish_stock
                        elif stock == (self.small_cap_stock or self.mid_cap_stock ):
                            self.short_stock  = self.shortMidCap            
                        elif stock == (self.xiv):
                            self.short_stock  = self.vxx  

                        if previousShort <> self.short_stock:
                            if VERBOSEMODE: print 'sell all positions of previousShort' , previousShort.Symbol
                            self.place_order(previousShort, 0) # don't want to keep holding other short
                        allocation[lastPos] = allocation[i] * 0.1  * -1 # how much to allocate to newly designated short position
                        if VERBOSEMODE: self.Log( 'shorting last position by this ratio{}' .format(allocation[lastPos]))                          

                        allocation[i]= allocation[i] + allocation[lastPos] # whichever was max allocation now get diminished by amount of allocation to short_stock
                        if VERBOSEMODE: self.Log('reducing max position to new lesser value of {}'. format(allocation[i]))                          
                        
                        break

            else:
                return
        except:
            print('An unknown error occurred in reallocateToShort.'), sys.exc_info()[0]   
            return           
            
            
    def trade(self):
        pass
            
    def allocSPY (self):  
        pass
            
    def variance(self,x,*args):
        try:
            p = np.squeeze(np.asarray(args))
            print "p squeezed is " ,p
            Acov = np.cov(p.T)
            r = np.dot(x,np.dot(Acov,x))
            #if VERBOSEMODE: self.Log( "Acov of p is {} result is {}".format( Acov,r))
            
            return r
        except:     
            print('An unknown error occurred trying to find variance.'), sys.exc_info()[0]   
            return 0.0



        
    def jac_variance(self,x,*args):
        try:
            p = np.squeeze(np.asarray(args))
            Acov = np.cov(p.T)

            return 2*np.dot(Acov,x)
        except:     
            self.Log('An unknown error occurred trying to find jacvariance.'), sys.exc_info()[0]   
            return 0.0
        

    def update_newFrame(self):  
        try:
            if VERBOSEMODE: self.Log('line 854 update_newFrame reached at date{} hour {} minute {}'.format(self.sliceTime.day,self.sliceTime.hour,self.sliceTime.minute)) 
            
            self.cash = self.Portfolio.get_Cash() 
            self.portvalue =  self.Portfolio.get_TotalPortfolioValue()

            self.totalLongs=0  
            self.totalShorts=0  
            #if VERBOSEMODE: self.Log("line 861")

         
            for s in self.Securities.Keys:
                if VERBOSEMODE: self.Log(' looping through self.Securities.Keys symbol {} quantity:{} average price{} '. format(s,self.Portfolio[s].Quantity , self.Portfolio[s].AveragePrice))
                if self.Portfolio[s].Quantity <0 :
                    self.totalShorts += (self.Portfolio[s].Quantity * self.Portfolio[s].AveragePrice)  
                else:
                    self.totalLongs += (self.Portfolio[s].Quantity * self.Portfolio[s].AveragePrice)  
            #if VERBOSEMODE: self.Log(' total longs {} total shorts {} ' . format(self.totalLongs , self.totalShorts))
            
 
            self.update_portvals()  
            #Handle assigning the timestep number (1 day is 1 timestep)  
            if VERBOSEMODE: self.Log( 'got to 874')
            if datetime.today().day <> self.day: #look for a new day  
                self.day=datetime.today().day  
                self.timestep += 1  
                #print ( "Cash: "+str(self.cash)+"; Margin Req: "+str(self.margin_req)+" Avail Cash:"+str(self.cash - self.margin_req) )  
                if VERBOSEMODE: self.Log(' excess liquidity {} ' . format(self.excess_liquidity))
                if self.excess_liquidity < 0 : 
                    self.generate_marginCall() 
            if VERBOSEMODE: self.Log('successfully ran update_newFrame')
        except:
            if VERBOSEMODE: self.Log('An unknown error occurred in update_newFrame.'), sys.exc_info()[0]   
            return      

    
    def wait(self, symbol=None, action=None):
        try:
            if VERBOSEMODE: self.Log("reached line 365 in wait for symbol {}" .format(symbol) )
            self.waits
        except NameError as e:
            if VERBOSEMODE: self.Log('A Name error occurred in wait .{} \ncreating waits array'. format(e) )   
            self.waits = {}
            return
        except:
            if VERBOSEMODE: self.Log('A known error occurred in wait.{}'. format(sys.exc_info()[0]) )   
            if VERBOSEMODE and symbol is not None: self.Log("reached line 369 creating waits array  for symbol {}" .format(symbol) )
            self.waits = {}
            return
        try:
            if symbol and action:
                if action == 1:
                    self.waits[symbol] = 1    # start wait
                elif action == 0:
                    del self.waits[symbol]    # end wait
            else:
                for symbol in self.waits.copy():
                    if self.waits[symbol] > self.waits_max:
                        del self.waits[symbol]
                    else:
                        self.waits[symbol] += 1   # increment
            if VERBOSEMODE: self.Log("reached line 378 {}" .format(self.waits[symbol]) )
        except:
            if VERBOSEMODE: self.Log('An unknown error occurred in wait.{}'. format(sys.exc_info()[0]) )   
                 

    def generate_marginCall(self):  
        try:
            #This function should be coded to address margin calls  
            self.Log("Margin call imminent so reallocating: Cash: {}  Margin Requirement: {}  Excess Margin:" .format(self.cash,self.margin_req,self.total_equity - self.margin_req))  
            self.Log("Margin Remaining per QC object {} ".format(self.Securities.MarginRemaining))
            #following causes a reallocation immediately
            self.is_trading_day = True
            self.allocate()
        except:
            if VERBOSEMODE: self.Log('An unknown error occurred in generate_marginCall {}'. format(sys.exc_info()[0]) )   
                 
    def update_portvals(self):  
        try:
            if VERBOSEMODE: self.Log('line 821 update_portvals reached at date{} hour {} minute {}'.format(self.sliceTime.day,self.sliceTime.hour,self.sliceTime.minute)) 

            #Update account information when this function is called  
            self.total_equity = self.cash + self.portvalue 

            if self.total_equity == 0:
                self.pct_invested = 0
            else:
                self.pct_invested = (self.totalLongs-self.totalShorts) / self.total_equity  
            self.pct_cash = self.cash / self.total_equity  
            if VERBOSEMODE: self.Log('self.pct_cash is {}'. format(self.pct_cash))
            #self.margin_req = abs(self.totalShorts * self.maint_margin)   
            self.margin_req = abs(decimal.Decimal(self.totalLongs) * 0.5 + decimal.Decimal(self.totalShorts) * 0.3 )
            if VERBOSEMODE: self.Log('self.margin_req is {}'. format(self.margin_req))

            self.excess_liquidity = decimal.Decimal(self.total_equity) - decimal.Decimal(self. margin_req)
            if VERBOSEMODE: self.Log('self.excess_liquidity is {}'. format(self.excess_liquidity))
            
           
            if VERBOSEMODE: self.Log('successfully ran update portvals')
        except:
            if VERBOSEMODE: self.Log('An unknown error occurred in update_portvals.'), sys.exc_info()[0]   
  

    

    def take_profit(self):    # Close some positions to take profit
        if VERBOSEMODE: self.Log('line 800 reached at date{} hour {} minute {}'.format(self.sliceTime.day,self.sliceTime.hour,self.sliceTime.minute))       
        pass
            
            
    def LimitOrder(self,sym,pct,limitprice):
        #replace this later with REAL limit order which will take absolute value not a percentage
        self.SetHoldings(sym,pct,True)

    

    def limit_losses(self):    
        pass



        

    def slope(self,in_list):     # Return slope of regression line. [Make sure this list contains no nans]
        return sm.OLS(in_list, sm.add_constant(range(-len(in_list) + 1, 1))).fit().params[-1]  # slope
    



    def get_history_by_symbol(self,symbol,period,resolution=None,attr=None):
        # what makes fetching history so complicated is that for some functions QC requires a string, for others a symbol
        # even though these funcitons are closely related.
        # history is fetched with Symbol type, Securities and Portfolio enumerations with string type. Latter are needed
        # to check if that symbol is already part of the searchable universe
        if VERBOSEMODE: self.Log('line 1051 get_history_by_symbol reached') 
        if RESEARCHMODE is None:
            self.Log('fatal error you must specify RESEARCHMODE as global variable so this function works in Research')
            return None
        emptyDF = pd.DataFrame() #creates a new dataframe that's empty
        if VERBOSEMODE: self.Log('line 1054 created dataframe') 

        try:
            #quantopian s history method allows symbol as object or string but not QC so we need to always 
            # convert to symbol before calling history method to streamline compatibility
            
            try:
                if isinstance(symbol,str):
                    sym = symbol     
                    if RESEARCHMODE:  
                        qb = QuantBook()                      
                        symbol = qb.AddEquity(sym).Symbol
                    else:
                        symbol = self.AddEquity(sym).Symbol
                else:
                    # must be in Symbol format already, so create a string variant
                    sym= str(symbol)
                if VERBOSEMODE: self.Log('line 1063 after transformation  in get_history_by_symbol types are sym:symbol  {}{}'. format(type(sym),type(symbol))) 
            except:
                if VERBOSEMODE: self.Log('An unknown error occurred trying to convert a symbol to a string type is {} Exiting '. format(type(symbol)))   
                return


            if attr is None:
                attr = "close"
            if resolution is None:
                resolution = "Resolution.Daily"            
            if RESEARCHMODE:
                # fails if I try to empty existing QB by setting to None
                history = qb.History(period, resolution)    
            else:
                #history will not work if this symbol not already in Portfolio object of 'self'
                try:
                   amt = self.Portfolio[sym].Quantity
                   if VERBOSEMODE: self.Log('line 1080 found symbol {} with quantity {}'.format(sym,amt) ) 
                except:
                    self.AddEquity(sym,resolution)
                    if VERBOSEMODE: self.Log('line 1085 added symbol {} to universe as no quantity found'.format(sym)  ) 
                        
                if VERBOSEMODE: self.Log('line 1101 testing history for symbol {} of type {} period {} resolution {}'.format(sym,type(symbol),period,resolution)  ) 
                try:
                    history = self.History(symbol,period,resolution) # watchout here you need Symbol not string 'sym'
                    if VERBOSEMODE: self.Log('line 1088  reached after fetching history of length {}'. format(len(history)) )
                    if len(history) == 0:
                        if VERBOSEMODE: self.Log('line 1181 no values found for history of symbol {}'. format(sym) )
                        return emptyDF
                except:       
                    self.Log('An unknown error occurred trying get history by_symbol line 1107' +  str(sys.exc_info()[0]) )                 
            df_history = history[attr].unstack(level=0).dropna()
            if VERBOSEMODE: self.Log('line 1096 in get_history_by_symbol succesffully created df_history of type {}'. format(type(df_history)) )

            return df_history

        except NameError as e:
                self.Log('A Name error occurred trying get history by_symbol.{}' .format(e) )
        except:       
            self.Log('An unknown error occurred trying get history by_symbol.' +  str(sys.exc_info()[0]) )
            return emptyDF        


    def get_history_from_list(self,stocklist,period,resolution = None, attr = None):
        if VERBOSEMODE: self.Log('line 1012 get_history_from_list reached') 


        try:
            emptyDF = pd.DataFrame() #creates a new dataframe that's empty
            if VERBOSEMODE: self.Log('line 1115 able to create empty dataframe') 
            
                
            
            if RESEARCHMODE is None:
                self.Log('fatal error you must specify RESEARCHMODE as global variable')
                return None
            if attr is None:
                attr = "close"
            if resolution is None:
                resolution = "Resolution.Daily"            
            if RESEARCHMODE:
                qb = None  # kill any exising instance to 'empty' symbollist
                qb = QuantBook() # recreate it   
                qb.SetStartDate(self.StartDate)
                for s in stocklist:
                    sym =  s.ToString()
                    qb.AddEquity(sym,resolution) # adds equity to our newly created quantbook universe
                    history = qb.History(period, resolution)
            else:
                strlist= []
                for s in stocklist:
                    strlist.append(s.ToString())
                
                history =  self.History(strlist,period, resolution)# will get all stocks in list of strings 
                if VERBOSEMODE: self.Log('line 1178 history from string list of type {}'. format(type(history) )) 
            df_history = history[attr].unstack(level=0).dropna()
            if VERBOSEMODE: self.Log('line 1180 created df_history_from list of type {}'. format(type(df_history)) )
            
            return df_history
        except:       
            self.Log('An unknown error occurred trying get history from symbol.' +  str(sys.exc_info()[0]) )            
            return emptyDF
            
    def symbolAsString(self,symbol):
        try:
            if isinstance(symbol,Securities.Equity.Equity):
                sym = symbol.Symbol.ToString()
            else:
                if isinstance(symbol,Symbol):
                    sym= str(symbol)
                else:
                    sym = symbol
            if VERBOSEMODE: self.Log("in symbolAsString successfully returned string {} ". format(sym))
                   
            return sym
        except:       
            self.Log('An unknown error occurred trying symbolAsString.' +  str(sys.exc_info()[0]) )            
            return ""
            
    
    def final_reporting(self):
        if is_date(2017, 1, 25):
            print("Total trades: %d" % self.total_trades)    

    def record_leverage(self):
        record(leverage = self.account.leverage)
        record(mx_lvrg = self.mx_lvrg)
        self.is_trading_day = False
        self.is_first_of_week = False  
    
   
        
    def testHistory(self,sym,period,resolution):
        try:
            if RESEARCHMODE:
                print 'researchmode enabled'
    
               
            else:
                try:
                    #make sure it exists in our securities list first
                    price = Securities[sym].Close
                except:
                    equity=self.AddEquity(sym,resolution)

                self.Log('line 1016 reached')

                self.Log('line 1018 reached') 
                history = self.History(equity,period,resolution)
                self.Log(str(history))   
                self.Log(str(len(history)))
                self.Log('line 1022 reached')                             
                             
            df_history = history["close"].unstack(level=0).dropna()
            return df_history
        except :
            self.Log('An unknown error occurred trying get history from list.' +  str(sys.exc_info()[0]) )
 
    def to_dataframe(data_c):
        # Helper function to make a dataframe from the coarse object. 
        # Then we can use all the handy dataframe methods.
        # Set the 'coerce_float' parameter to get meaningful datatypes
        # otherwise all the fields will be un-useful generic objects.
        data = [(
                stock.Price, 
                stock.Volume, 
                stock.DollarVolume, 
                stock.HasFundamentalData)
                for stock in data_c]
                
        symbols = [stock.Symbol for stock in data_c ]
        labels = ['price', 'volume', 'dollar_volume', 'has_fundamentals']
        
        data_df = pd.DataFrame.from_records(
                data, 
                index=symbols, 
                columns=labels, 
                coerce_float=True)
                
        return data_df
        
    # create and return list of all holdings
    def getPortfolioHoldings(self):
       # get the symbols of our holding stocks
        holding_list = []
        for i in self.Portfolio:
            if i.Value.Invested:
                holding_list.append(i.Value.Symbol)
        return holding_list