//============================================================================ //TRTH REST API Learning Tutorial 5: main program code //Goal: a partially productized On Demand extraction request: // instrument lists: create a large instrument list array outside of DSS, // populated from an input file, with error handling; // field lists: create a field list array, outside of DSS; // create and run an on demand extraction for intraday summaries, wait for it to complete; // save data in compressed file; // display received data and extraction notes on screen; // include some error checking and reporting. // Cleanup is not required in this case. //Limit: this code does not take into account the maximum size of TRTH // instrument lists (30000 instruments). //For Gzip decompression we use SharpZipLib. //============================================================================ //To facilitate the learning process, the member declarations are at the start, //followed by the main and finally the methods. //============================================================================ using System; using System.Collections.Generic; using System.IO; using System.Text; using ICSharpCode.SharpZipLib.GZip; using ThomsonReuters.Dss.Api.Extractions; using ThomsonReuters.Dss.Api.Content; using ThomsonReuters.Dss.Api.Extractions.ReportTemplates; using ThomsonReuters.Dss.Api.Extractions.ExtractionRequests; using ThomsonReuters.Dss.Api.Core; using DssRestfulApiTutorials; namespace DssRestfulApiTutorials { class Program { //===================================================================== //Member declarations //===================================================================== private static string dssUserName = "YourID"; private static string dssUserPassword = "YourPassword"; private static string outputDirectory = "C:\\API TRTH REST\\"; private static string dataOutputFile = outputDirectory + "TRTH_API_tutorial_5l_data.csv.gz"; private static string noteOutputFile = outputDirectory + "TRTH_API_tutorial_5l_note.txt"; private static string errorOutputFile = outputDirectory + "TRTH_API_tutorial_5l_errors.txt"; private static string inputDirectory = outputDirectory; private static string instrumentIdentifiersInputFile = inputDirectory + "TRTH_API_5_input_file.csv"; //File format: 1 instrument per line. //Line format: , , //Example lines, first two are for a Cusip (CSP) and a Reuters Instrument Code (RIC): //CSP,31352JRV1 //RIC,ALVG.DE //CHR,0#.FCHI,CAC40 //CIN,G93882192,Vodaphone //COM,001179594,Carrefour //ISN,CH0012138530,CS //SED,B1YW440,3i Group //VAL,24476758,UBS //WPK,A11Q05,Elumeo private static int maxDataLines = 1000000; //Limit the number of lines of data we display on screen //===================================================================== //Main program entry //===================================================================== static void Main() { //----------------------------------------------------------------- //Connect and authenticate to the DSS server: //----------------------------------------------------------------- DssClient dssClient = new DssClient(); dssClient.ConnectToServer(dssUserName, dssUserPassword); DebugPrintAndWaitForEnter("Returned session token: " + dssClient.SessionToken); //Expose the extractionsContext from dssClient: ExtractionsContext extractionsContext = dssClient.extractionsContext; //----------------------------------------------------------------- //Check if the input file exists: //----------------------------------------------------------------- if (!File.Exists(instrumentIdentifiersInputFile)) { DebugPrintAndWaitForEnter("FATAL: cannot access " + instrumentIdentifiersInputFile + "\nCheck if file and directory exist."); return; //Exit main program } //----------------------------------------------------------------- //Check if the output directory exists: //----------------------------------------------------------------- if (!Directory.Exists(outputDirectory)) { DebugPrintAndWaitForEnter("FATAL: cannot access directory " + outputDirectory + "\nCheck if it exists."); return; //Exit main program } //----------------------------------------------------------------- //Empty output files of data from previous run: //----------------------------------------------------------------- StreamWriter sw = new StreamWriter(dataOutputFile, false); sw.Close(); Console.WriteLine("Cleared data output file " + dataOutputFile + "\n"); sw = new StreamWriter(noteOutputFile, false); sw.Close(); Console.WriteLine("Cleared note output file " + dataOutputFile + "\n"); sw = new StreamWriter(errorOutputFile, false); sw.Close(); Console.WriteLine("Cleared error output file " + errorOutputFile + "\n"); //----------------------------------------------------------------- //Now we move away from controlling the GUI via the API, //to directly interfacing with the API. //This is why, contrary to the previous tutorials, we do not create //an instrument list and report template on the DSS server. //----------------------------------------------------------------- //----------------------------------------------------------------- //Create an array outside of DSS, populate it with instruments, //read from an input file, with error handling and logging: //----------------------------------------------------------------- //Initialise the error output file: sw = new StreamWriter(errorOutputFile, true); sw.WriteLine("INFO: list of errors found in input file: " + instrumentIdentifiersInputFile + ":"); //Open the input file: StreamReader sr = new StreamReader(instrumentIdentifiersInputFile); //Instead of an array (of defined length), create a list for the instrument identifiers. //We do this because: // we don't know how many instruments are in the file, // we filter the file entries, to only keep the validated ones. //Once the list is filled, we convert it to an array, as that is what the API needs. //Create the empty instrument identifiers list: List instrumentIdentifiersList = new List (); //Populate the list, reading one line at a time from the file: int fileLineNumber = 0; string fileLine = string.Empty; string identifierTypeString = string.Empty; string identifierCodeString = string.Empty; IdentifierType identifierType; int i = 0; //Loop through all lines until we get to the end of the file: bool endOfFile = false; while (!endOfFile) { //Read one line of the file, test if end of file: fileLine = sr.ReadLine(); endOfFile = (fileLine == null); if (!endOfFile) { fileLineNumber++; try { //Parse the file line to extract the comma separated instrument type and code: string[] splitLine = fileLine.Split(new char[] { ',' }); identifierTypeString = splitLine[0]; identifierCodeString = splitLine[1]; //Add only validated instrument identifiers into our list. if (identifierTypeString != string.Empty) { if (identifierCodeString != string.Empty) { //DSS can handle many types, here we only handle a subset: switch (identifierTypeString) { case "CHR": identifierType = IdentifierType.ChainRIC; break; case "CIN": identifierType = IdentifierType.Cin; break; case "COM": identifierType = IdentifierType.CommonCode; break; case "CSP": identifierType = IdentifierType.Cusip; break; case "ISN": identifierType = IdentifierType.Isin; break; case "RIC": identifierType = IdentifierType.Ric; break; case "SED": identifierType = IdentifierType.Sedol; break; case "VAL": identifierType = IdentifierType.Valoren; break; case "WPK": identifierType = IdentifierType.Wertpapier; break; default: identifierType = IdentifierType.NONE; DebugPrintAndWriteToFile("ERROR: line " + fileLineNumber + ": unknown identifier type: " + identifierTypeString, sw); break; } if (identifierType != IdentifierType.NONE) { instrumentIdentifiersList.Add(new InstrumentIdentifier { IdentifierType = identifierType, Identifier = identifierCodeString }); Console.WriteLine("Line " + fileLineNumber + ": " + identifierTypeString + " " + identifierCodeString + " loaded into array [" + i + "]"); i++; } } //Error handling messages grouped here: else { DebugPrintAndWriteToFile("ERROR: line " + fileLineNumber + ": missing identifier code in line: " + fileLine, sw); } } else { DebugPrintAndWriteToFile("ERROR: line " + fileLineNumber + ": missing identifier type in line: " + fileLine, sw); } } catch { DebugPrintAndWriteToFile("ERROR: line " + fileLineNumber + ": bad line format: " + fileLine, sw); } } else { if (fileLineNumber == 0) { DebugPrintAndWriteToFile("ERROR: empty file: " + instrumentIdentifiersInputFile, sw); } } } //End of while loop sr.Close(); int validIdentifiersCount = i; if (validIdentifiersCount == 0) { DebugPrintAndWriteToFile("\nFATAL: program exit due to no valid identifiers in the list.", sw); DebugPrintAndWaitForEnter(""); sw.Close(); return; //Exit main program } sw.Close(); Console.WriteLine("\n" + validIdentifiersCount + " valid instruments were loaded into an array, outside of DSS,\n" + "for use in the extraction."); //Convert the instrument identifiers list to an array: InstrumentIdentifier[] instrumentIdentifiers = instrumentIdentifiersList.ToArray(); //----------------------------------------------------------------- //Create an array of field names: //----------------------------------------------------------------- string[] requestedFieldNames = CreateRequestedFieldNames(); //----------------------------------------------------------------- //Define the conditions: //----------------------------------------------------------------- var requestDate = new DateTime(2017, 7, 27); var utcStartDate = requestDate.AddHours(8); var utcEndDate = requestDate.AddHours(16); var startDate = new DateTimeOffset(utcStartDate); var endDate = new DateTimeOffset(utcEndDate); TickHistoryTimeAndSalesCondition condition = new TickHistoryTimeAndSalesCondition { ReportDateRangeType = ReportDateRangeType.Range, QueryStartDate = startDate, QueryEndDate = endDate, ApplyCorrectionsAndCancellations = false, ExtractBy = TickHistoryExtractByMode.Ric, MessageTimeStampIn = TickHistoryTimeOptions.GmtUtc, SortBy = TickHistorySort.SingleByRic, DisplaySourceRIC = true }; //----------------------------------------------------------------- //Create and run an on demand extraction: //----------------------------------------------------------------- //As an example we choose to do an intraday summary (bar data) extraction: DebugPrintAndWaitForEnter("We also created an array of field names, outside of DSS." + "\n\nNext we will launch a direct on demand intraday summary extraction," + "\nusing these 2 arrays."); Console.WriteLine("{0:T} Please be patient and wait for the extraction to complete ...\n", DateTime.Now); //Create an on demand extraction, of type TickHistoryIntradaySummariesExtractionRequest, //using our instrument identifiers and requested field names arrays: TickHistoryTimeAndSalesExtractionRequest extractionRequest = new TickHistoryTimeAndSalesExtractionRequest { IdentifierList = InstrumentIdentifierList.Create(instrumentIdentifiers), ContentFieldNames = requestedFieldNames, Condition = condition }; //Extraction //NOTE: if the extraction request takes more than 30 seconds the async mechanism will be used, //the SDK handles this automatically for you. //Do not automatically decompress data file content: extractionsContext.Options.AutomaticDecompression = false; RawExtractionResult extractionResult = extractionsContext.ExtractRaw(extractionRequest); Console.Write("{0:T} Extraction complete ... ", DateTime.Now); DebugPrintAndWaitForEnter(""); //----------------------------------------------------------------- //Data treatment starts here (we treat the Note file further down). //----------------------------------------------------------------- //We can simply save the compressed data, as a compressed file: //----------------------------------------------------------------- DssStreamResponse streamResponse = extractionsContext.GetReadStream(extractionResult); using (var fileStream = File.Create(dataOutputFile)) streamResponse.Stream.CopyTo(fileStream); Console.WriteLine("Saved the compressed data file: " + dataOutputFile); //----------------------------------------------------------------- //Alternative: decompress it on the fly, and treat it line by line: //----------------------------------------------------------------- Console.WriteLine("==================================== DATA ====================================="); streamResponse = extractionsContext.GetReadStream(extractionResult); { int lineCount = 0; string line = ""; using (GZipInputStream gzip = new GZipInputStream(streamResponse.Stream)) { using (StreamReader reader = new StreamReader(gzip, Encoding.UTF8)) { line = reader.ReadLine(); if (string.IsNullOrEmpty(line)) { Console.WriteLine("WARNING: no data returned. Check your request dates."); sw.WriteLine("WARNING: no data returned. Check your request dates."); } else { //The first line is the list of field names: Console.WriteLine(line); lineCount++; //The remaining lines are the data: //Variant 1 (no limit on screen output): write all lines to console //while ((line = reader.ReadLine()) != null) //{ // Console.WriteLine(line); // lineCount++; //} //Variant 2: write lines individually to console (with a limit on number of lines we output) for (lineCount = 1; !reader.EndOfStream; lineCount++) { line = reader.ReadLine(); if (lineCount < maxDataLines) { Console.WriteLine(line); }; if (lineCount == maxDataLines) { Console.WriteLine("==============================================================================="); Console.WriteLine("Stop screen output now, we reached the max number of lines we set for display:\n" + maxDataLines); Console.WriteLine("Counting the remaining lines, please be patient ..."); } } Console.WriteLine("Total line count for data: " + lineCount); } } } } DebugPrintAndWaitForEnter("==============================================================================="); //----------------------------------------------------------------- //Notes file: output to screen, and parse to analyse content: //----------------------------------------------------------------- sw = new StreamWriter(noteOutputFile, true); Console.WriteLine("==================================== NOTES ===================================="); Boolean success = false; string errorMsgs = ""; string warningMsgs = ""; string permissionMsgs = ""; foreach (String notes in extractionResult.Notes) { sw.WriteLine(notes); Console.WriteLine(notes); //The returned notes are in a single string. To analyse the contents line by line we split it: string[] notesLines = notes.Split(new string[] { "\n", "\r\n" }, StringSplitOptions.RemoveEmptyEntries); foreach (string notesLine in notesLines) { success = success || (notesLine.Contains("Processing completed successfully")); if (notesLine.Contains("ERROR")) { errorMsgs = errorMsgs + notesLine + "\n"; } if (notesLine.Contains("WARNING")) { warningMsgs = warningMsgs + notesLine + "\n"; } if (notesLine.Contains("row suppressed for lack of")) { permissionMsgs = permissionMsgs + notesLine + "\n"; } } } sw.Close(); DebugPrintAndWaitForEnter("==============================================================================="); if (success) { Console.WriteLine("SUCCESS: processing completed successfully.\n"); } else { errorMsgs = "ERROR: processing did not complete successfully !\n" + errorMsgs; } sw = new StreamWriter(errorOutputFile, true); if (errorMsgs != "") { sw.WriteLine("\nERROR messages in notes file:\n" + errorMsgs); Console.WriteLine("ERROR messages:\n" + errorMsgs); } if (warningMsgs != "") { sw.WriteLine("WARNING messages in notes file:\n" + warningMsgs); Console.WriteLine("WARNING messages:\n" + warningMsgs); } if (permissionMsgs != "") { sw.WriteLine("PERMISSION ISSUES messages in notes file:\n" + permissionMsgs); Console.WriteLine("PERMISSION ISSUES messages:\n" + permissionMsgs); } sw.Close(); DebugPrintAndWaitForEnter("==============================================================================="); } //===================================================================== //Helper methods //===================================================================== static void DebugPrintAndWaitForEnter(string messageToPrint) { Console.WriteLine(messageToPrint); Console.WriteLine("Press Enter to continue"); Console.ReadLine(); } static void DebugPrintAndWriteToFile(string messageToPrintAndWriteToFile, StreamWriter sw) { Console.WriteLine(messageToPrintAndWriteToFile); sw.WriteLine(messageToPrintAndWriteToFile); } static string[] CreateRequestedFieldNames() { string[] requestedFieldNames = { "Trade - Bid Price", "Trade - Price", "Trade - Ask Price", "Trade - Volume", "Trade - Exchange Time" }; return requestedFieldNames; } } }