# -*- coding: utf-8 -*-


import requests
import json
import shutil
import time
import gzip
import os
import pandas as pd

#Set these parameters before running the code:
filePath = "C:/Users/atyagi1/Documents"  #Location to save downloaded files
#filePath="//Nighthawkxa/data$/EcmReporting/Quant_MarketData/Refinitiv"
#filePath="//Nighthawkxa/data$/EcmReporting/Quant_MarketData/"
#filePath="//Nighthawk/data$/GEIBQuant/"
dataDir = "/refinitiv/"
#dataDir = "/DATA/"
moveDir = "backup/"
fileName = "refinitiv_intraday_tick_data"
myUsername = "9026556"
myPassword = "RJET1234"
useAws = False
#Set the last parameter above to:
# False to download from TRTH servers
# True to download from Amazon Web Services cloud (recommended, it is faster)
fileNameGZ = filePath + dataDir + fileName + ".csv.gz"
fileNameOut = filePath + dataDir + fileName + ".csv"

def getReutersToken():
    #Step 1: token request
 
    requestUrl = "https://selectapi.datascope.refinitiv.com/RestApi/v1/Authentication/RequestToken"
 
    requestHeaders={
        "Prefer":"respond-async",
        "Content-Type":"application/json"
        }
 
    requestBody={
        "Credentials": {
        "Username": myUsername,
        "Password": myPassword
      }
    }
    r1 = requests.post(requestUrl, json=requestBody, headers=requestHeaders,verify=False)
    if r1.status_code == 200:
        return r1.json()
    else:
        print(r1.json())
       # print("Authentication Error Status Code: "+ str(r1.status_code) +" Message:"+r1.text)
        return None
    
if __name__ == "__main__":
    reuters_response = getReutersToken()
    if not reuters_response:
        print("Check token generation code, maybe invalid json")
    token = reuters_response["value"]
    print(token)
    
    #Step 2: send an on demand extraction request using the received token

    requestUrl='https://selectapi.datascope.refinitiv.com/RestApi/v1/Extractions/ExtractRaw'

    requestHeaders={
        "Prefer":"respond-async",
        "Content-Type":"application/json",
        "Authorization": "token " + token
    }

    requestBody={
      "ExtractionRequest": {
        "@odata.type": "#DataScope.Select.Api.Extractions.ExtractionRequests.TickHistoryTimeAndSalesExtractionRequest",
        #"@odata.type":"#DataScope.Select.Api.Extractions.ExtractionRequests.TickHistoryMarketDepthExtractionRequest",   
        "ContentFieldNames": [
          "Trade - Price", 
          "Trade - Volume", 
          "Quote-Bid Price",
          "Quote-Bid Size",
          "Quote-Ask Price",
          "Quote-Ask Size",
          "Trade-Bid Price",
          "Trade-Ask Price",
          "Trade-Sequence Number",
          "Trade-Qualifier"
          "Trade - Exchange Time"

        ],
        "IdentifierList": {
          "@odata.type": "#DataScope.Select.Api.Extractions.ExtractionRequests.InstrumentIdentifierList",
          "InstrumentIdentifiers": [{
            "Identifier": "MSFT.OQ",
            "IdentifierType": "Ric"
          },{
            "Identifier": "AAPL.OQ",
            "IdentifierType": "Ric"
          }],
          "UseUserPreferencesForValidationOptions":"false"
        },

        "Condition": {
      #    "MessageTimeStampIn": "LocalExchangeTime",        #GmtUtc
            "MessageTimeStampIn": "GmtUtc",
            "ReportDateRangeType": "Range",
            "QueryStartDate": "2021-11-09T08:45:00.000Z",
            "QueryEndDate": "2021-11-09T12:30:00.000Z",
            "DisplaySourceRIC": "True"
         # "ReportDateRangeType": "Relative",                #Range
         # "RelativeStartDaysAgo": 3,
        #  "RelativeStartTime": "09:25:00.000",
        #  "RelativeEndDaysAgo": 1,
       #   "RelativeEndTime": "16:15:00.000",
       #   "TimeRangeMode": "Inclusive",
    #     "QueryStartDate": "2021-10-26T09:25:00.000",
    #     "QueryEndDate": "2021-10-26T09:35:00.000",
    #     "DateRangeTimeZone": "Local Exchange Time Zone" #"Eastern Time (US & Canada)", #"UTC-05:00",  #"Local Exchange Time Zone" # localExcTimeZone gives error on both REst APi and GUI
      #    "SummaryInterval": "OneMinute",
        #  "TimebarPersistence":"true",
         # "DisplaySourceRIC":"true",
         # "View":"NormalizedLL2",
         # "NumberOfLevels":10,
         # "SortBy":"SingleByRic",
         # "MessageTimeStampIn":"GmtUtc",
         # "ReportDateRangeType":"Range",
         # "QueryStartDate":"2017-07-01T00:00:00Z",
         # "QueryEndDate":"2017-08-23T00:00:00Z",
         # "Preview":"None",
         # "ExtractBy":"Ric",
         # "DisplaySourceRIC":"true",
        }
      }
    }
    multiRICFile = "C:\\Users\\atyagi1\\Documents\\Refinitiv_data\\symb_listS.txt"
    with open(multiRICFile) as file:
        ricsList = [line.strip() for line in file]
    print(ricsList)
    ricsExpandedList = []
    for ric in ricsList:
        ricsExpandedList.append({'Identifier': ric, 'IdentifierType': 'Ric'})
    print(ricsExpandedList)   
    ###
    print(requestBody['ExtractionRequest']['IdentifierList']['InstrumentIdentifiers'])
    requestBody['ExtractionRequest']['IdentifierList']['InstrumentIdentifiers'] = ricsExpandedList
    r2 = requests.post(requestUrl, json=requestBody,headers=requestHeaders, verify=False)

    #Display the HTTP status of the response
    #Initial response status (after approximately 30 seconds wait) is usually 202
    status_code = r2.status_code
    print (time.strftime("%H:%M:%S"), "HTTP status of the response: " + str(status_code))
    
    #print(r2.json())
    
    # Step 3: If required, poll the status of the request using the received location URL. Once the request has completed, retrieve the jobId and extraction notes.

    #If status is 202, display the location url we received, and will use to poll the status of the extraction request:
    if status_code == 202 :
        requestUrl = r2.headers["location"]
        print (time.strftime("%H:%M:%S"), 'Extraction is not complete, we shall poll the location URL:', str(requestUrl))

        requestHeaders={
            "Prefer":"respond-async",
            "Content-Type":"application/json",
            "Authorization":"token " + token
        }

        r3 = requests.get(requestUrl, headers=requestHeaders,verify=False)
        #As long as the status of the request is 202, the extraction is not finished;
        #we must wait, and poll the status until it is no longer 202:
        status_code = r3.status_code

        while (status_code == 202):
            print (time.strftime("%H:%M:%S"), 'As we received a 202, we wait 20 seconds, then poll again (until we receive a 200)')
            time.sleep(20)
            r3 = requests.get(requestUrl,headers=requestHeaders, verify=False)
            status_code = r3.status_code
            print (time.strftime("%H:%M:%S"), 'HTTP status of the response: ' + str(status_code))

            #When the status of the request is 200 the extraction is complete;
            #we retrieve and display the jobId and the extraction notes (it is recommended to analyse their content)):

    if status_code == 200 :
        r3Json = r3.json()
        jobId = r3Json["JobId"]
        # print ('\njobId: ' + jobId + '\n')
        notes = r3Json["Notes"]
        print (time.strftime("%H:%M:%S"), 'Extraction notes:\n' + notes[0])

    #If instead of a status 200 we receive a different status, there was an error:
    if status_code != 200 :
        print (time.strftime("%H:%M:%S"), 'An error occured. Try to run this cell again. If it fails, re-run the previous cell.\n')
        
    # Step 4: get the extraction results, using the received jobId. We also save the compressed data to disk, as a GZIP.

    requestUrl = "https://selectapi.datascope.refinitiv.com/RestApi/v1/Extractions/RawExtractionResults" + "('" + jobId + "')" + "/$value"

    #AWS requires an additional header: X-Direct-Download
    if useAws:
        requestHeaders={
            "Prefer":"respond-async",
            "Content-Type":"text/plain",
            "Accept-Encoding":"gzip",
            "X-Direct-Download":"true",
            "Authorization": "token " + token
        }
    else:
        requestHeaders={
            "Prefer":"respond-async",
            "Content-Type":"text/plain",
            "Accept-Encoding":"gzip",
            "Authorization": "token " + token
        }

    r5 = requests.get(requestUrl,headers=requestHeaders,stream=True,verify=False)
    #Ensure we do not automatically decompress the data on the fly:
    r5.raw.decode_content = False
    if useAws:
        print (time.strftime("%H:%M:%S"), 'Content response headers (AWS server): type: ' + r5.headers["Content-Type"] + '\n')
        #AWS does not set header Content-Encoding="gzip".
    else:
        print (time.strftime("%H:%M:%S"), 'Content response headers (TRTH server): type: ' + r5.headers["Content-Type"] + ' - encoding: ' + r5.headers["Content-Encoding"] + '\n')

    rr = r5.raw
    
   # with gzip.open(r5.raw, "rb") as raw_gzip_file:
      #  print(raw_gzip_file.read())
       # raw_gzip_file.write(f_out)
   
   # gzip.write()
    print (time.strftime("%H:%M:%S"), 'Saving compressed data to file:' + fileNameGZ + ' ... please be patient')
    chunk_size = 1024
   # rr = r5.raw
    with open(fileNameGZ, 'wb') as fd:
        shutil.copyfileobj(rr, fd, chunk_size)
    #fd.close

    print (time.strftime("%H:%M:%S"), 'Finished saving compressed data to file:' + fileNameGZ + '\n')

    #with gzip.open(fileNameGZ, 'r') as f_in, open(fileNameOut, 'wb') as f_out:
     # shutil.copyfileobj(f_in, f_out)

   # if os.path.exists(fileNameGZ):
   #   os.remove(fileNameGZ)
   # print (time.strftime("%H:%M:%S"), 'Deleted compressed file:' + fileNameGZ + '\n')
  #  return fileNameOut