From 0ca66e3218b9e253296500103e4cb37f6a6b4f72 Mon Sep 17 00:00:00 2001 From: Anthony Aufdenkampe Date: Thu, 14 Apr 2022 14:43:03 -0500 Subject: [PATCH 1/8] Setup for HMAC authentication capabilities @SRGDamia1, general HMAC functions could benefit all dataPublishers, so I am adding it to the dataPublisherBase. Does that make sense? Once I get general HMAC SHA256 tokens to work, I'll then be creating a new publisher for Azure EventHubs. AWS IoT has a similar endpoint, so this could be widely used. --- library.json | 12 +++++++++++- src/dataPublisherBase.cpp | 8 ++++++++ src/dataPublisherBase.h | 14 ++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/library.json b/library.json index b6a5c1262..36af1f319 100644 --- a/library.json +++ b/library.json @@ -329,6 +329,16 @@ "authors": ["Sara Damiano", "Anthony Aufdenkampe"], "frameworks": "arduino", "platforms": "atmelavr, atmelsam" - } + }, + { + "name": "cryptosuite2", + "owner": "envirodiy", + "url": "https://github.com/EnviroDIY/cryptosuite2", + "version": "~0.2.7", + "note": "Arduino/Generic C library for SHA256, SHA1 hashing and SHA256-HMAC, SHA1-HMAC", + "authors": [], + "frameworks": "arduino", + "platforms": "*" + } ] } diff --git a/src/dataPublisherBase.cpp b/src/dataPublisherBase.cpp index cf5529747..93bf3d350 100644 --- a/src/dataPublisherBase.cpp +++ b/src/dataPublisherBase.cpp @@ -155,3 +155,11 @@ String dataPublisher::parseMQTTState(int state) { default: return String(state) + ": UNKNOWN"; } } + + +String dataPublisher::writeHMACsignature(char* key, char* string_to_sign) { + // Create a HexMap to save 16 bytes of SRAM + const char hexMap[] PROGMEM = "0123456789abcdef"; // This is from the cryptosuite2 example, but there must be a better way. + // Anthony to add more code here + return signature; +} diff --git a/src/dataPublisherBase.h b/src/dataPublisherBase.h index daf6fd494..80e9262a4 100644 --- a/src/dataPublisherBase.h +++ b/src/dataPublisherBase.h @@ -52,6 +52,7 @@ #undef MS_DEBUGGING_STD #include "LoggerBase.h" #include "Client.h" +#include "sha256.h" // `cryptosuite2` library's SHA256 functions /** * @brief The dataPublisher class is a virtual class used by other publishers to @@ -268,6 +269,19 @@ class dataPublisher { */ String parseMQTTState(int state); + /** + * @brief Write an HMAC-SHA256 signature -- which is a keyed-hash message + * authentication code (HMAC) created using the SHA-256 cryptographic + * hash algorithm -- for generating tokens for authenticating requests + * using the authorization header. + * + * @param key The shared secret key used to "salt" the hash + * @param string_to_sign The string that gets hashed into a signature token. + * @return **String** The signed HMAC-SHA256 authorization token, or + * signature. + */ + String writeHMACsignature(char* key, char* string_to_sign); + protected: /** From b0da816d4355feef3ce2185c7dc60916c924be53 Mon Sep 17 00:00:00 2001 From: Anthony Aufdenkampe Date: Fri, 15 Apr 2022 13:49:24 -0500 Subject: [PATCH 2/8] Add HMACEventHubTest & Publisher (not yet working) --- .../HMACEventHubTest/HMACEventHubTest.ino | 369 ++++++++++++++++++ sensor_tests/HMACEventHubTest/ReadMe.md | 44 +++ sensor_tests/HMACEventHubTest/platformio.ini | 35 ++ src/dataPublisherBase.cpp | 23 +- src/dataPublisherBase.h | 4 +- src/publishers/EventHubPublisher.cpp | 302 ++++++++++++++ src/publishers/EventHubPublisher.h | 246 ++++++++++++ 7 files changed, 1016 insertions(+), 7 deletions(-) create mode 100644 sensor_tests/HMACEventHubTest/HMACEventHubTest.ino create mode 100644 sensor_tests/HMACEventHubTest/ReadMe.md create mode 100644 sensor_tests/HMACEventHubTest/platformio.ini create mode 100644 src/publishers/EventHubPublisher.cpp create mode 100644 src/publishers/EventHubPublisher.h diff --git a/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino b/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino new file mode 100644 index 000000000..39abf9ae4 --- /dev/null +++ b/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino @@ -0,0 +1,369 @@ +/** ========================================================================= + * @file HMACEventHubTest.ino + * @brief Test, based off DRWI_SIM7080LTE.ino + * + * This example shows proper settings for the following configuration: + * + * Mayfly v1.1 board + * EnviroDIY SIM7080 LTE module (with Hologram SIM card) + * + * @author Sara Geleskie Damiano + * @copyright (c) 2017-2022 Stroud Water Research Center (SWRC) + * and the EnviroDIY Development Team + * This example is published under the BSD-3 license. + * + + * Hardware Platform: EnviroDIY Mayfly Arduino Datalogger + * + * DISCLAIMER: + * THIS CODE IS PROVIDED "AS IS" - NO WARRANTY IS GIVEN. + * ======================================================================= */ + +// ========================================================================== +// Defines for the Arduino IDE +// NOTE: These are ONLY needed to compile with the Arduino IDE. +// If you use PlatformIO, you should set these build flags in your +// platformio.ini +// ========================================================================== +/** Start [defines] */ +#ifndef TINY_GSM_RX_BUFFER +#define TINY_GSM_RX_BUFFER 64 +#endif +#ifndef TINY_GSM_YIELD_MS +#define TINY_GSM_YIELD_MS 2 +#endif +/** End [defines] */ + +// ========================================================================== +// Include the libraries required for any data logger +// ========================================================================== +/** Start [includes] */ +// The Arduino library is needed for every Arduino program. +#include + +// EnableInterrupt is used by ModularSensors for external and pin change +// interrupts and must be explicitly included in the main program. +#include + +// Include the main header for ModularSensors +#include +/** End [includes] */ + + +// ========================================================================== +// Data Logging Options +// ========================================================================== +/** Start [logging_options] */ +// The name of this program file +const char* sketchName = "HMACEventHubTest.ino"; +// Logger ID, also becomes the prefix for the name of the data file on SD card +const char* LoggerID = "XXXXX"; +// How frequently (in minutes) to log data +const uint8_t loggingInterval = 1; +// Your logger's timezone. +const int8_t timeZone = -5; // Eastern Standard Time +// NOTE: Daylight savings time will not be applied! Please use standard time! + +// Set the input and output pins for the logger +// NOTE: Use -1 for pins that do not apply +const int32_t serialBaud = 115200; // Baud rate for debugging +const int8_t greenLED = 8; // Pin for the green LED +const int8_t redLED = 9; // Pin for the red LED +const int8_t buttonPin = 21; // Pin for debugging mode (ie, button pin) +const int8_t wakePin = 31; // MCU interrupt/alarm pin to wake from sleep +// Mayfly 0.x D31 = A7 +const int8_t sdCardPwrPin = -1; // MCU SD card power pin +const int8_t sdCardSSPin = 12; // SD card chip select/slave select pin +const int8_t sensorPowerPin = 22; // MCU pin controlling main sensor power +/** End [logging_options] */ + + +// ========================================================================== +// Wifi/Cellular Modem Options +// ========================================================================== +/** Start [sim_com_sim7080] */ +// For almost anything based on the SIMCom SIM7080G +#include + +// Create a reference to the serial port for the modem +HardwareSerial& modemSerial = Serial1; // Use hardware serial if possible +const int32_t modemBaud = 9600; // SIM7080 does auto-bauding by default, but + // for simplicity we set to 9600 + +// Modem Pins - Describe the physical pin connection of your modem to your board +// NOTE: Use -1 for pins that do not apply + +const int8_t modemVccPin = 18; +// MCU pin controlling modem power --- Pin 18 is the power enable pin for the +// bee socket on Mayfly v1.0, use -1 if using Mayfly 0.5b or if the bee socket +// is constantly powered (ie you changed SJ18 on Mayfly 1.x to 3.3v) +const int8_t modemStatusPin = 19; // MCU pin used to read modem status +const int8_t modemSleepRqPin = 23; // MCU pin for modem sleep/wake request +const int8_t modemLEDPin = redLED; // MCU pin connected an LED to show modem + // status + +// Network connection information +const char* apn = + "hologram"; // APN connection name, typically Hologram unless you have a + // different provider's SIM card. Change as needed + +// Create the modem object +SIMComSIM7080 modem7080(&modemSerial, modemVccPin, modemStatusPin, + modemSleepRqPin, apn); +// Create an extra reference to the modem by a generic name +SIMComSIM7080 modem = modem7080; +/** End [sim_com_sim7080] */ + + +// ========================================================================== +// Using the Processor as a Sensor +// ========================================================================== +/** Start [processor_sensor] */ +#include + +// Create the main processor chip "sensor" - for general metadata +const char* mcuBoardVersion = "v1.1"; +ProcessorStats mcuBoard(mcuBoardVersion); +/** End [processor_sensor] */ + + +// ========================================================================== +// Maxim DS3231 RTC (Real Time Clock) +// ========================================================================== +/** Start [ds3231] */ +#include + +// Create a DS3231 sensor object +MaximDS3231 ds3231(1); +/** End [ds3231] */ + + + +// ========================================================================== +// Creating the Variable Array[s] and Filling with Variable Objects +// ========================================================================== +/** Start [variable_arrays] */ +Variable* variableList[] = { + new ProcessorStats_Battery(&mcuBoard), + new MaximDS3231_Temp(&ds3231), + new Modem_SignalPercent(&modem), +}; + +// All UUID's, device registration, and sampling feature information can be +// pasted directly from Monitor My Watershed. +// To get the list, click the "View token UUID list" button on the upper right +// of the site page. + +// *** CAUTION --- CAUTION --- CAUTION --- CAUTION --- CAUTION *** +// Check the order of your variables in the variable list!!! +// Be VERY certain that they match the order of your UUID's! +// Rearrange the variables in the variable list ABOVE if necessary to match! +// Do not change the order of the variables in the section below. +// *** CAUTION --- CAUTION --- CAUTION --- CAUTION --- CAUTION *** + +// Replace all of the text in the following section with the UUID array from +// MonitorMyWatershed + +/* clang-format off */ +// --------------------- Beginning of Token UUID List --------------------- + + +const char* UUIDs[] = // UUID array for device sensors + { + "12345678-abcd-1234-ef00-1234567890ab", // Battery voltage (EnviroDIY_Mayfly_Batt) + "12345678-abcd-1234-ef00-1234567890ab", // Battery voltage (EnviroDIY_Mayfly_Batt) + "12345678-abcd-1234-ef00-1234567890ab", // Percent full scale (EnviroDIY_LTEB_SignalPercent) +}; +const char* registrationToken = "12345678-abcd-1234-ef00-1234567890ab"; // Device registration token +const char* samplingFeature = "12345678-abcd-1234-ef00-1234567890ab"; // Sampling feature UUID + + +// ----------------------- End of Token UUID List ----------------------- +/* clang-format on */ + +// Count up the number of pointers in the array +int variableCount = sizeof(variableList) / sizeof(variableList[0]); + +// Create the VariableArray object +VariableArray varArray(variableCount, variableList, UUIDs); +/** End [variable_arrays] */ + + +// ========================================================================== +// The Logger Object[s] +// ========================================================================== +/** Start [loggers] */ +// Create a new logger instance +Logger dataLogger(LoggerID, loggingInterval, &varArray); +/** End [loggers] */ + + +// ========================================================================== +// Creating Data Publisher[s] +// ========================================================================== +/** Start [publishers] */ +// Create a data publisher for the Monitor My Watershed/EnviroDIY POST endpoint +#include +EnviroDIYPublisher EnviroDIYPOST(dataLogger, &modem.gsmClient, + registrationToken, samplingFeature); +/** End [publishers] */ + + +// ========================================================================== +// Working Functions +// ========================================================================== +/** Start [working_functions] */ +// Flashes the LED's on the primary board +void greenredflash(uint8_t numFlash = 4, uint8_t rate = 75) { + for (uint8_t i = 0; i < numFlash; i++) { + digitalWrite(greenLED, HIGH); + digitalWrite(redLED, LOW); + delay(rate); + digitalWrite(greenLED, LOW); + digitalWrite(redLED, HIGH); + delay(rate); + } + digitalWrite(redLED, LOW); +} + +// Reads the battery voltage +// NOTE: This will actually return the battery level from the previous update! +float getBatteryVoltage() { + if (mcuBoard.sensorValues[0] == -9999) mcuBoard.update(); + return mcuBoard.sensorValues[0]; +} + + +// ========================================================================== +// Arduino Setup Function +// ========================================================================== +/** Start [setup] */ +void setup() { + // Start the primary serial connection + Serial.begin(serialBaud); + + // Print a start-up note to the first serial port + Serial.print(F("Now running ")); + Serial.print(sketchName); + Serial.print(F(" on Logger ")); + Serial.println(LoggerID); + Serial.println(); + + Serial.print(F("Using ModularSensors Library version ")); + Serial.println(MODULAR_SENSORS_VERSION); + Serial.print(F("TinyGSM Library version ")); + Serial.println(TINYGSM_VERSION); + Serial.println(); + + // Start the serial connection with the modem + modemSerial.begin(modemBaud); + + // Set up pins for the LED's + pinMode(greenLED, OUTPUT); + digitalWrite(greenLED, LOW); + pinMode(redLED, OUTPUT); + digitalWrite(redLED, LOW); + // Blink the LEDs to show the board is on and starting up + greenredflash(); + + pinMode(20, OUTPUT); // for proper operation of the onboard flash memory + // chip's ChipSelect (Mayfly v1.0 and later) + + // Set the timezones for the logger/data and the RTC + // Logging in the given time zone + Logger::setLoggerTimeZone(timeZone); + // It is STRONGLY RECOMMENDED that you set the RTC to be in UTC (UTC+0) + Logger::setRTCTimeZone(0); + + // Attach the modem and information pins to the logger + dataLogger.attachModem(modem); + modem.setModemLED(modemLEDPin); + dataLogger.setLoggerPins(wakePin, sdCardSSPin, sdCardPwrPin, buttonPin, + greenLED); + + // Begin the logger + dataLogger.begin(); + + // Test HMAC token + // Secret key and Plain Text to Compute Hash + const char* key = "Jefe"; + const char* text_to_hash = "what do ya want for nothing?"; + + // Call method of dataPublisher object + EnviroDIYPOST.writeHMACtoken(key, text_to_hash); + + + // Note: Please change these battery voltages to match your battery + // Set up the sensors, except at lowest battery level + if (getBatteryVoltage() > 3.4) { + Serial.println(F("Setting up sensors...")); + varArray.setupSensors(); + } + + /** Start [setup_sim7080] */ + modem.setModemWakeLevel(HIGH); // ModuleFun Bee inverts the signal + modem.setModemResetLevel(HIGH); // ModuleFun Bee inverts the signal + Serial.println(F("Waking modem and setting Cellular Carrier Options...")); + modem.modemWake(); // NOTE: This will also set up the modem + modem.gsmModem.setBaud(modemBaud); // Make sure we're *NOT* auto-bauding! + modem.gsmModem.setNetworkMode(38); // set to LTE only + // 2 Automatic + // 13 GSM only + // 38 LTE only + // 51 GSM and LTE only + modem.gsmModem.setPreferredMode(1); // set to CAT-M + // 1 CAT-M + // 2 NB-IoT + // 3 CAT-M and NB-IoT + /** End [setup_sim7080] */ + + + // Sync the clock if it isn't valid or we have battery to spare + if (getBatteryVoltage() > 3.55 || !dataLogger.isRTCSane()) { + // Synchronize the RTC with NIST + // This will also set up the modem + dataLogger.syncRTC(); + } + + // Create the log file, adding the default header to it + // Do this last so we have the best chance of getting the time correct and + // all sensor names correct + // Writing to the SD card can be power intensive, so if we're skipping + // the sensor setup we'll skip this too. + if (getBatteryVoltage() > 3.4) { + Serial.println(F("Setting up file on SD card")); + dataLogger.turnOnSDcard( + true); // true = wait for card to settle after power up + dataLogger.createLogFile(true); // true = write a new header + dataLogger.turnOffSDcard( + true); // true = wait for internal housekeeping after write + } + + // Call the processor sleep + Serial.println(F("Putting processor to sleep\n")); + dataLogger.systemSleep(); +} +/** End [setup] */ + + +// ========================================================================== +// Arduino Loop Function +// ========================================================================== +/** Start [loop] */ +// Use this short loop for simple data logging and sending +void loop() { + // Note: Please change these battery voltages to match your battery + // At very low battery, just go back to sleep + if (getBatteryVoltage() < 3.4) { + dataLogger.systemSleep(); + } + // At moderate voltage, log data but don't send it over the modem + else if (getBatteryVoltage() < 3.55) { + dataLogger.logData(); + } + // If the battery is good, send the data to the world + else { + dataLogger.logDataAndPublish(); + } +} +/** End [loop] */ diff --git a/sensor_tests/HMACEventHubTest/ReadMe.md b/sensor_tests/HMACEventHubTest/ReadMe.md new file mode 100644 index 000000000..0f9e164c2 --- /dev/null +++ b/sensor_tests/HMACEventHubTest/ReadMe.md @@ -0,0 +1,44 @@ +# DRWI Sites with EnviroDIY LTE Bees +Example sketch for using the EnviroDIY SIM7080G LTE cellular module with an EnviroDIY Mayfly Data Logger. + +The exact hardware configuration used in this example: + * Mayfly v1.0 board + * EnviroDIY SIM7080 LTE module (with Hologram SIM card) + * Hydros21 CTD sensor + * Campbell Scientific OBS3+ Turbidity sensor + +An EnviroDIY LTE SIM7080 module can be used with the older Mayfly v0.5b boards if you change line 101 (for modemVccPin) from 18 to -1. +This is because the Mayfly v1.0 board has a separate 3.3v regulator to power the Bee socket and is controlled by turning pin 18 on or off. +Mayfly v0.5b has the Bee socket constantly powered, therefore using "-1" is the proper setting for that line of code. + +The EnviroDIY LTE SIM7080 module includes 2 antennas in the package. The small thin one is the cellular antenna, and should be connected to the socket labeled "CELL". The thicker block is the GPS antenna, and should be connected to the "GPS" socket, but only if you intend to use the GPS functionality of the module. ModularSensors does not currently suport GPS functionality, but other libraries such as TinyGPS can work with the SIM7080 module. + +The included cell antenna works best in high-signal-strength areas. For most remote areas and logger deployments, we suggest a larger LTE antenna, like the W3907B0100 +from PulseLarsen (Digikey 1837-1003-ND or Mouser 673-W3907B0100) + +_______ + +[//]: # ( @tableofcontents ) + +[//]: # ( @m_footernavigation ) + +[//]: # ( Start GitHub Only ) +- [DRWI Sites with EnviroDIY LTE Bees](#drwi-sites-with-envirodiy-lte-bees) +- [Unique Features of the DRWI EnviroDIY LTE Example](#unique-features-of-the-drwi-envirodiy-lte-example) + +[//]: # ( End GitHub Only ) + +_______ + +# Unique Features of the DRWI EnviroDIY LTE Example +- Specifically for sites within the Delaware River Watershed Initiative. +- Uses a EnviroDIY LTE Bee based on the SIMCom SIM7080G + + +[//]: # ( @section example_drwi_ediylte_pio_config PlatformIO Configuration ) + +[//]: # ( @include{lineno} DRWI_SIM7080LTE/platformio.ini ) + +[//]: # ( @section example_drwi_ediylte_code The Complete Code ) + +[//]: # ( @include{lineno} DRWI_SIM7080LTE/DRWI_SIM7080LTE.ino ) diff --git a/sensor_tests/HMACEventHubTest/platformio.ini b/sensor_tests/HMACEventHubTest/platformio.ini new file mode 100644 index 000000000..8acac309a --- /dev/null +++ b/sensor_tests/HMACEventHubTest/platformio.ini @@ -0,0 +1,35 @@ +; PlatformIO Project Configuration File +; +; Build options: build flags, source filter +; Upload options: custom upload port, speed and extra flags +; Library options: dependencies, extra library storages +; Advanced options: extra scripting +; +; Please visit documentation for the other options and examples +; http://docs.platformio.org/page/projectconf.html + +[platformio] +description = ModularSensors example intended for DRWI users with CTD, turbidity, and a EnviroDIY SIM7080G LTE modem + +[env:mayfly] +monitor_speed = 57600 +board = mayfly +platform = atmelavr +framework = arduino +lib_ldf_mode = deep+ +lib_ignore = + RTCZero + Adafruit NeoPixel + Adafruit GFX Library + Adafruit SSD1306 + Adafruit ADXL343 + Adafruit STMPE610 + Adafruit TouchScreen + Adafruit ILI9341 +build_flags = + -DSDI12_EXTERNAL_PCINT +lib_deps = + envirodiy/EnviroDIY_ModularSensors +; ^^ Use this when working from an official release of the library +; https://github.com/EnviroDIY/ModularSensors.git#develop +; ^^ Use this when if you want to pull from the develop branch diff --git a/src/dataPublisherBase.cpp b/src/dataPublisherBase.cpp index 93bf3d350..a1515d9d4 100644 --- a/src/dataPublisherBase.cpp +++ b/src/dataPublisherBase.cpp @@ -157,9 +157,22 @@ String dataPublisher::parseMQTTState(int state) { } -String dataPublisher::writeHMACsignature(char* key, char* string_to_sign) { - // Create a HexMap to save 16 bytes of SRAM - const char hexMap[] PROGMEM = "0123456789abcdef"; // This is from the cryptosuite2 example, but there must be a better way. - // Anthony to add more code here - return signature; +String dataPublisher::writeHMACtoken(const char* key, const char* string_to_sign) { + MS_DBG(F("Writing an HMAC-SHA256 token")); + // Anthony to add more code here + // Initialize the Sha256Wrapper for HMAC hashing, "salting" with the secret key + // Must be invoked before hashing. + Sha256.initHmac((uint8_t*)key, strlen(key)); // Recasting char* key to uint8_t* + // Write data into the hasher. + Sha256.write(string_to_sign); // equivalent to `Sha256.print()` + // Returns a reference to the hash. Once this method has been called init must be invoked again. + uint8_t* result = Sha256.resultHmac(); + // Print 32-byte hash as hexidecimal string (64 characters) + Serial.print(F("Token as Hex: ")); + for (int i = 0; i < 32; i++) { + if (result[i] < 16) Serial.print("0"); // Otherwise leading zeros will be dropped + Serial.print(result[i],HEX); + } + Serial.print(F("\n\n")); + return "token"; // eventually return token } diff --git a/src/dataPublisherBase.h b/src/dataPublisherBase.h index 80e9262a4..24b9679ba 100644 --- a/src/dataPublisherBase.h +++ b/src/dataPublisherBase.h @@ -270,7 +270,7 @@ class dataPublisher { String parseMQTTState(int state); /** - * @brief Write an HMAC-SHA256 signature -- which is a keyed-hash message + * @brief Write an HMAC-SHA256 token -- which is a keyed-hash message * authentication code (HMAC) created using the SHA-256 cryptographic * hash algorithm -- for generating tokens for authenticating requests * using the authorization header. @@ -280,7 +280,7 @@ class dataPublisher { * @return **String** The signed HMAC-SHA256 authorization token, or * signature. */ - String writeHMACsignature(char* key, char* string_to_sign); + String writeHMACtoken(const char* key, const char* string_to_sign); protected: diff --git a/src/publishers/EventHubPublisher.cpp b/src/publishers/EventHubPublisher.cpp new file mode 100644 index 000000000..ddc5e714b --- /dev/null +++ b/src/publishers/EventHubPublisher.cpp @@ -0,0 +1,302 @@ +/** + * @file EventHubPublisher.cpp + * @copyright 2017-2022 Stroud Water Research Center + * Part of the EnviroDIY ModularSensors library for Arduino + * @author Sara Geleskie Damiano + * @author Anthony Aufdenkampe + * + * @brief Implements the EventHubPublisher class. + */ + +#include "EventHubPublisher.h" + + +// ============================================================================ +// Functions for the EnviroDIY data portal receivers. +// ============================================================================ + +// Constant values for post requests +// I want to refer to these more than once while ensuring there is only one copy +// in memory +const char* EventHubPublisher::postEndpoint = "/api/data-stream/"; +const char* EventHubPublisher::enviroDIYHost = "data.envirodiy.org"; +const int EventHubPublisher::enviroDIYPort = 80; +const char* EventHubPublisher::tokenHeader = "\r\nTOKEN: "; +// const unsigned char *EventHubPublisher::cacheHeader = "\r\nCache-Control: +// no-cache"; const unsigned char *EventHubPublisher::connectionHeader = +// "\r\nConnection: close"; +const char* EventHubPublisher::contentLengthHeader = "\r\nContent-Length: "; +const char* EventHubPublisher::contentTypeHeader = + "\r\nContent-Type: application/json\r\n\r\n"; + +const char* EventHubPublisher::samplingFeatureTag = "{\"sampling_feature\":\""; +const char* EventHubPublisher::timestampTag = "\",\"timestamp\":\""; + + +// Constructors +EventHubPublisher::EventHubPublisher() : dataPublisher() { + // MS_DBG(F("dataPublisher object created")); + _registrationToken = NULL; +} +EventHubPublisher::EventHubPublisher(Logger& baseLogger, uint8_t sendEveryX, + uint8_t sendOffset) + : dataPublisher(baseLogger, sendEveryX, sendOffset) { + // MS_DBG(F("dataPublisher object created")); + _registrationToken = NULL; +} +EventHubPublisher::EventHubPublisher(Logger& baseLogger, Client* inClient, + uint8_t sendEveryX, uint8_t sendOffset) + : dataPublisher(baseLogger, inClient, sendEveryX, sendOffset) { + // MS_DBG(F("dataPublisher object created")); +} +EventHubPublisher::EventHubPublisher(Logger& baseLogger, + const char* registrationToken, + const char* samplingFeatureUUID, + uint8_t sendEveryX, uint8_t sendOffset) + : dataPublisher(baseLogger, sendEveryX, sendOffset) { + setToken(registrationToken); + _baseLogger->setSamplingFeatureUUID(samplingFeatureUUID); + // MS_DBG(F("dataPublisher object created")); +} +EventHubPublisher::EventHubPublisher(Logger& baseLogger, Client* inClient, + const char* registrationToken, + const char* samplingFeatureUUID, + uint8_t sendEveryX, uint8_t sendOffset) + : dataPublisher(baseLogger, inClient, sendEveryX, sendOffset) { + setToken(registrationToken); + _baseLogger->setSamplingFeatureUUID(samplingFeatureUUID); + // MS_DBG(F("dataPublisher object created")); +} +// Destructor +EventHubPublisher::~EventHubPublisher() {} + + +void EventHubPublisher::setToken(const char* registrationToken) { + _registrationToken = registrationToken; + // MS_DBG(F("Registration token set!")); +} + + +// Calculates how long the JSON will be +uint16_t EventHubPublisher::calculateJsonSize() { + uint16_t jsonLength = 21; // {"sampling_feature":" + jsonLength += 36; // sampling feature UUID + jsonLength += 15; // ","timestamp":" + jsonLength += 25; // markedISO8601Time + jsonLength += 2; // ", + for (uint8_t i = 0; i < _baseLogger->getArrayVarCount(); i++) { + jsonLength += 1; // " + jsonLength += 36; // variable UUID + jsonLength += 2; // ": + jsonLength += _baseLogger->getValueStringAtI(i).length(); + if (i + 1 != _baseLogger->getArrayVarCount()) { + jsonLength += 1; // , + } + } + jsonLength += 1; // } + + return jsonLength; +} + + +/* +// Calculates how long the full post request will be, including headers +uint16_t EventHubPublisher::calculatePostSize() +{ + uint16_t postLength = 31; // "POST /api/data-stream/ HTTP/1.1" + postLength += 28; // "\r\nHost: data.envirodiy.org" + postLength += 11; // "\r\nTOKEN: " + postLength += 36; // registrationToken + // postLength += 27; // "\r\nCache-Control: no-cache" + // postLength += 21; // "\r\nConnection: close" + postLength += 20; // "\r\nContent-Length: " + postLength += String(calculateJsonSize()).length(); + postLength += 42; // "\r\nContent-Type: application/json\r\n\r\n" + postLength += calculateJsonSize(); + return postLength; +} +*/ + + +// This prints a properly formatted JSON for EnviroDIY to an Arduino stream +void EventHubPublisher::printSensorDataJSON(Stream* stream) { + stream->print(samplingFeatureTag); + stream->print(_baseLogger->getSamplingFeatureUUID()); + stream->print(timestampTag); + stream->print( + _baseLogger->formatDateTime_ISO8601(Logger::markedLocalEpochTime)); + stream->print(F("\",")); + + for (uint8_t i = 0; i < _baseLogger->getArrayVarCount(); i++) { + stream->print('"'); + stream->print(_baseLogger->getVarUUIDAtI(i)); + stream->print(F("\":")); + stream->print(_baseLogger->getValueStringAtI(i)); + if (i + 1 != _baseLogger->getArrayVarCount()) { stream->print(','); } + } + + stream->print('}'); +} + + +// This prints a fully structured post request for EnviroDIY to the +// specified stream. +void EventHubPublisher::printEnviroDIYRequest(Stream* stream) { + // Stream the HTTP headers for the post request + stream->print(postHeader); + stream->print(postEndpoint); + stream->print(HTTPtag); + stream->print(hostHeader); + stream->print(enviroDIYHost); + stream->print(tokenHeader); + stream->print(_registrationToken); + // stream->print(cacheHeader); + // stream->print(connectionHeader); + stream->print(contentLengthHeader); + stream->print(calculateJsonSize()); + stream->print(contentTypeHeader); + + // Stream the JSON itself + printSensorDataJSON(stream); +} + + +// A way to begin with everything already set +void EventHubPublisher::begin(Logger& baseLogger, Client* inClient, + const char* registrationToken, + const char* samplingFeatureUUID) { + setToken(registrationToken); + dataPublisher::begin(baseLogger, inClient); + _baseLogger->setSamplingFeatureUUID(samplingFeatureUUID); +} +void EventHubPublisher::begin(Logger& baseLogger, + const char* registrationToken, + const char* samplingFeatureUUID) { + setToken(registrationToken); + dataPublisher::begin(baseLogger); + _baseLogger->setSamplingFeatureUUID(samplingFeatureUUID); +} + + +// This utilizes an attached modem to make a TCP connection to the +// EnviroDIY/ODM2DataSharingPortal and then streams out a post request +// over that connection. +// The return is the http status code of the response. +// int16_t EventHubPublisher::postDataEnviroDIY(void) +int16_t EventHubPublisher::publishData(Client* outClient) { + // Create a buffer for the portions of the request and response + char tempBuffer[37] = ""; + uint16_t did_respond = 0; + + MS_DBG(F("Outgoing JSON size:"), calculateJsonSize()); + + // Open a TCP/IP connection to the Enviro DIY Data Portal (WebSDL) + MS_DBG(F("Connecting client")); + MS_START_DEBUG_TIMER; + if (outClient->connect(enviroDIYHost, enviroDIYPort)) { + MS_DBG(F("Client connected after"), MS_PRINT_DEBUG_TIMER, F("ms\n")); + + // copy the initial post header into the tx buffer + strcpy(txBuffer, postHeader); + strcat(txBuffer, postEndpoint); + strcat(txBuffer, HTTPtag); + + // add the rest of the HTTP POST headers to the outgoing buffer + // before adding each line/chunk to the outgoing buffer, we make sure + // there is space for that line, sending out buffer if not + if (bufferFree() < 28) printTxBuffer(outClient); + strcat(txBuffer, hostHeader); + strcat(txBuffer, enviroDIYHost); + + if (bufferFree() < 47) printTxBuffer(outClient); + strcat(txBuffer, tokenHeader); + strcat(txBuffer, _registrationToken); + + // if (bufferFree() < 27) printTxBuffer(outClient); + // strcat(txBuffer, cacheHeader); + + // if (bufferFree() < 21) printTxBuffer(outClient); + // strcat(txBuffer, connectionHeader); + + if (bufferFree() < 26) printTxBuffer(outClient); + strcat(txBuffer, contentLengthHeader); + itoa(calculateJsonSize(), tempBuffer, 10); // BASE 10 + strcat(txBuffer, tempBuffer); + + if (bufferFree() < 42) printTxBuffer(outClient); + strcat(txBuffer, contentTypeHeader); + + // put the start of the JSON into the outgoing response_buffer + if (bufferFree() < 21) printTxBuffer(outClient); + strcat(txBuffer, samplingFeatureTag); + + if (bufferFree() < 36) printTxBuffer(outClient); + strcat(txBuffer, _baseLogger->getSamplingFeatureUUID()); + + if (bufferFree() < 42) printTxBuffer(outClient); + strcat(txBuffer, timestampTag); + _baseLogger->formatDateTime_ISO8601(Logger::markedLocalEpochTime) + .toCharArray(tempBuffer, 37); + strcat(txBuffer, tempBuffer); + txBuffer[strlen(txBuffer)] = '"'; + txBuffer[strlen(txBuffer)] = ','; + + for (uint8_t i = 0; i < _baseLogger->getArrayVarCount(); i++) { + // Once the buffer fills, send it out + if (bufferFree() < 47) printTxBuffer(outClient); + + txBuffer[strlen(txBuffer)] = '"'; + _baseLogger->getVarUUIDAtI(i).toCharArray(tempBuffer, 37); + strcat(txBuffer, tempBuffer); + txBuffer[strlen(txBuffer)] = '"'; + txBuffer[strlen(txBuffer)] = ':'; + _baseLogger->getValueStringAtI(i).toCharArray(tempBuffer, 37); + strcat(txBuffer, tempBuffer); + if (i + 1 != _baseLogger->getArrayVarCount()) { + txBuffer[strlen(txBuffer)] = ','; + } else { + txBuffer[strlen(txBuffer)] = '}'; + } + } + + // Send out the finished request (or the last unsent section of it) + printTxBuffer(outClient, true); + + // Wait 10 seconds for a response from the server + uint32_t start = millis(); + while ((millis() - start) < 10000L && outClient->available() < 12) { + delay(10); + } + + // Read only the first 12 characters of the response + // We're only reading as far as the http code, anything beyond that + // we don't care about. + did_respond = outClient->readBytes(tempBuffer, 12); + + // Close the TCP/IP connection + MS_DBG(F("Stopping client")); + MS_RESET_DEBUG_TIMER; + outClient->stop(); + MS_DBG(F("Client stopped after"), MS_PRINT_DEBUG_TIMER, F("ms")); + } else { + PRINTOUT(F("\n -- Unable to Establish Connection to EnviroDIY Data " + "Portal --")); + } + + // Process the HTTP response + int16_t responseCode = 0; + if (did_respond > 0) { + char responseCode_char[4]; + for (uint8_t i = 0; i < 3; i++) { + responseCode_char[i] = tempBuffer[i + 9]; + } + responseCode = atoi(responseCode_char); + } else { + responseCode = 504; + } + + PRINTOUT(F("-- Response Code --")); + PRINTOUT(responseCode); + + return responseCode; +} diff --git a/src/publishers/EventHubPublisher.h b/src/publishers/EventHubPublisher.h new file mode 100644 index 000000000..8120eaaf5 --- /dev/null +++ b/src/publishers/EventHubPublisher.h @@ -0,0 +1,246 @@ +/** + * @file EventHubPublisher.h + * @copyright 2017-2022 Stroud Water Research Center + * Part of the EnviroDIY ModularSensors library for Arduino + * @author Sara Geleskie Damiano + * @author Anthony Aufdenkampe + * + * @brief Contains the EventHubPublisher subclass of dataPublisher for + * publishing data to the Monitor My Watershed/EnviroDIY data portal at + * http://data.enviroDIY.org + */ + +// Header Guards +#ifndef SRC_PUBLISHERS_EVENTHUBPUBLISHER_H_ +#define SRC_PUBLISHERS_EVENTHUBPUBLISHER_H_ + +// Debugging Statement +// #define MS_EVENTHUBPUBLISHER_DEBUG + +#ifdef MS_EVENTHUBPUBLISHER_DEBUG +#define MS_DEBUGGING_STD "EventHubPublisher" +#endif + +// Included Dependencies +#include "ModSensorDebugger.h" +#undef MS_DEBUGGING_STD +#include "dataPublisherBase.h" + + +// ============================================================================ +// Functions for the EnviroDIY data portal receivers. +// ============================================================================ +/** + * @brief The EventHubPublisher subclass of dataPublisher for publishing data + * to the Monitor My Watershed/EnviroDIY data portal at + * http://data.enviroDIY.org + * + * @ingroup the_publishers + */ +class EventHubPublisher : public dataPublisher { + public: + // Constructors + /** + * @brief Construct a new EnviroDIY Publisher object with no members set. + */ + EventHubPublisher(); + /** + * @brief Construct a new EnviroDIY Publisher object + * + * @note If a client is never specified, the publisher will attempt to + * create and use a client on a LoggerModem instance tied to the attached + * logger. + * + * @param baseLogger The logger supplying the data to be published + * @param sendEveryX Currently unimplemented, intended for future use to + * enable caching and bulk publishing + * @param sendOffset Currently unimplemented, intended for future use to + * enable publishing data at a time slightly delayed from when it is + * collected + * + * @note It is possible (though very unlikey) that using this constructor + * could cause errors if the compiler attempts to initialize the publisher + * instance before the logger instance. If you suspect you are seeing that + * issue, use the null constructor and a populated begin(...) within your + * set-up function. + */ + explicit EventHubPublisher(Logger& baseLogger, uint8_t sendEveryX = 1, + uint8_t sendOffset = 0); + /** + * @brief Construct a new EnviroDIY Publisher object + * + * @param baseLogger The logger supplying the data to be published + * @param inClient An Arduino client instance to use to print data to. + * Allows the use of any type of client and multiple clients tied to a + * single TinyGSM modem instance + * @param sendEveryX Currently unimplemented, intended for future use to + * enable caching and bulk publishing + * @param sendOffset Currently unimplemented, intended for future use to + * enable publishing data at a time slightly delayed from when it is + * collected + * + * @note It is possible (though very unlikey) that using this constructor + * could cause errors if the compiler attempts to initialize the publisher + * instance before the logger instance. If you suspect you are seeing that + * issue, use the null constructor and a populated begin(...) within your + * set-up function. + */ + EventHubPublisher(Logger& baseLogger, Client* inClient, + uint8_t sendEveryX = 1, uint8_t sendOffset = 0); + /** + * @brief Construct a new EnviroDIY Publisher object + * + * @param baseLogger The logger supplying the data to be published + * @param registrationToken The registration token for the site on the + * Monitor My Watershed data portal. + * @param samplingFeatureUUID The sampling feature UUID for the site on the + * Monitor My Watershed data portal. + * @param sendEveryX Currently unimplemented, intended for future use to + * enable caching and bulk publishing + * @param sendOffset Currently unimplemented, intended for future use to + * enable publishing data at a time slightly delayed from when it is + * collected + */ + EventHubPublisher(Logger& baseLogger, const char* registrationToken, + const char* samplingFeatureUUID, uint8_t sendEveryX = 1, + uint8_t sendOffset = 0); + /** + * @brief Construct a new EnviroDIY Publisher object + * + * @param baseLogger The logger supplying the data to be published + * @param inClient An Arduino client instance to use to print data to. + * Allows the use of any type of client and multiple clients tied to a + * single TinyGSM modem instance + * @param registrationToken The registration token for the site on the + * Monitor My Watershed data portal. + * @param samplingFeatureUUID The sampling feature UUID for the site on the + * Monitor My Watershed data portal. + * @param sendEveryX Currently unimplemented, intended for future use to + * enable caching and bulk publishing + * @param sendOffset Currently unimplemented, intended for future use to + * enable publishing data at a time slightly delayed from when it is + * collected + */ + EventHubPublisher(Logger& baseLogger, Client* inClient, + const char* registrationToken, + const char* samplingFeatureUUID, uint8_t sendEveryX = 1, + uint8_t sendOffset = 0); + /** + * @brief Destroy the EnviroDIY Publisher object + */ + virtual ~EventHubPublisher(); + + // Returns the data destination + String getEndpoint(void) override { + return String(enviroDIYHost); + } + + // Adds the site registration token + /** + * @brief Set the site registration token + * + * @param registrationToken The registration token for the site on the + * Monitor My Watershed data portal. + */ + void setToken(const char* registrationToken); + + /** + * @brief Calculates how long the outgoing JSON will be + * + * @return uint16_t The number of characters in the JSON object. + */ + uint16_t calculateJsonSize(); + // /** + // * @brief Calculates how long the full post request will be, including + // * headers + // * + // * @return uint16_t The length of the full request including HTTP + // headers. + // */ + // uint16_t calculatePostSize(); + + /** + * @brief This generates a properly formatted JSON for EnviroDIY and prints + * it to the input Arduino stream object. + * + * @param stream The Arduino stream to write out the JSON to. + */ + void printSensorDataJSON(Stream* stream); + + /** + * @brief This prints a fully structured post request for Monitor My + * Watershed/EnviroDIY to the specified stream. + * + * @param stream The Arduino stream to write out the request to. + */ + void printEnviroDIYRequest(Stream* stream); + + // A way to begin with everything already set + /** + * @copydoc dataPublisher::begin(Logger& baseLogger, Client* inClient) + * @param registrationToken The registration token for the site on the + * Monitor My Watershed data portal. + * @param samplingFeatureUUID The sampling feature UUID for the site on the + * Monitor My Watershed data portal. + */ + void begin(Logger& baseLogger, Client* inClient, + const char* registrationToken, const char* samplingFeatureUUID); + /** + * @copydoc dataPublisher::begin(Logger& baseLogger) + * @param registrationToken The registration token for the site on the + * Monitor My Watershed data portal. + * @param samplingFeatureUUID The sampling feature UUID for the site on the + * Monitor My Watershed data portal. + */ + void begin(Logger& baseLogger, const char* registrationToken, + const char* samplingFeatureUUID); + + // int16_t postDataEnviroDIY(void); + /** + * @brief Utilize an attached modem to open a a TCP connection to the + * EnviroDIY/ODM2DataSharingPortal and then stream out a post request over + * that connection. + * + * This depends on an internet connection already having been made and a + * client being available. + * + * @param outClient An Arduino client instance to use to print data to. + * Allows the use of any type of client and multiple clients tied to a + * single TinyGSM modem instance + * @return **int16_t** The http status code of the response. + */ + int16_t publishData(Client* outClient) override; + + protected: + /** + * @anchor envirodiy_post_vars + * @name Portions of the POST request to EnviroDIY + * + * @{ + */ + static const char* postEndpoint; ///< The endpoint + static const char* enviroDIYHost; ///< The host name + static const int enviroDIYPort; ///< The host port + static const char* tokenHeader; ///< The token header text + // static const char *cacheHeader; ///< The cache header text + // static const char *connectionHeader; ///< The keep alive header text + static const char* contentLengthHeader; ///< The content length header text + static const char* contentTypeHeader; ///< The content type header text + /**@}*/ + + /** + * @anchor envirodiy_json_vars + * @name Portions of the JSON object for EnviroDIY + * + * @{ + */ + static const char* samplingFeatureTag; ///< The JSON feature UUID tag + static const char* timestampTag; ///< The JSON feature timestamp tag + /**@}*/ + + private: + // Tokens and UUID's for EnviroDIY + const char* _registrationToken; +}; + +#endif // SRC_PUBLISHERS_EventHubPublisher_H_ From 5de0fa7a5e166a947af0b46ede2af8f7e570114e Mon Sep 17 00:00:00 2001 From: Anthony Aufdenkampe Date: Fri, 15 Apr 2022 16:08:27 -0500 Subject: [PATCH 3/8] EventHubPublisher almost working but still getting 504. Also, using SAS generated in Python for now. --- .../HMACEventHubTest/HMACEventHubTest.ino | 21 ++++++------ src/publishers/EventHubPublisher.cpp | 34 +++++++++---------- src/publishers/EventHubPublisher.h | 20 +++++------ 3 files changed, 38 insertions(+), 37 deletions(-) diff --git a/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino b/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino index 39abf9ae4..a094f3989 100644 --- a/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino +++ b/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino @@ -57,7 +57,7 @@ // The name of this program file const char* sketchName = "HMACEventHubTest.ino"; // Logger ID, also becomes the prefix for the name of the data file on SD card -const char* LoggerID = "XXXXX"; +const char* LoggerID = "HMACEventHubTest"; // How frequently (in minutes) to log data const uint8_t loggingInterval = 1; // Your logger's timezone. @@ -144,9 +144,9 @@ MaximDS3231 ds3231(1); // ========================================================================== /** Start [variable_arrays] */ Variable* variableList[] = { - new ProcessorStats_Battery(&mcuBoard), + // new ProcessorStats_Battery(&mcuBoard), new MaximDS3231_Temp(&ds3231), - new Modem_SignalPercent(&modem), + // new Modem_SignalPercent(&modem), }; // All UUID's, device registration, and sampling feature information can be @@ -170,12 +170,13 @@ Variable* variableList[] = { const char* UUIDs[] = // UUID array for device sensors { - "12345678-abcd-1234-ef00-1234567890ab", // Battery voltage (EnviroDIY_Mayfly_Batt) - "12345678-abcd-1234-ef00-1234567890ab", // Battery voltage (EnviroDIY_Mayfly_Batt) - "12345678-abcd-1234-ef00-1234567890ab", // Percent full scale (EnviroDIY_LTEB_SignalPercent) + // formatted below to all have 11 characters + // "Mayfly_Batt", // Battery voltage (EnviroDIY_Mayfly_Batt) + "Mayfly_Temp", // Board Temperature (EnviroDIY_Mayfly_Temp) + // "LTEB_Signal", // Percent full scale (EnviroDIY_LTEB_SignalPercent) }; -const char* registrationToken = "12345678-abcd-1234-ef00-1234567890ab"; // Device registration token -const char* samplingFeature = "12345678-abcd-1234-ef00-1234567890ab"; // Sampling feature UUID +const char* registrationToken = "SharedAccessSignature sr=https%3A%2F%2Fevent-hub-data-logger.servicebus.windows.net%2Fdevices%2Fmessages&sig=fHhAkb/uw/8W2DcVMYJePm0K4QVabKAv2CRE9j3Al1A%3D&se=1650074784&skn=mayfly-device"; // Device registration token +const char* samplingFeature = "7d37e135-0e26-4bc7-aa81-f9443283582d"; // Sampling feature UUID // ----------------------- End of Token UUID List ----------------------- @@ -203,8 +204,8 @@ Logger dataLogger(LoggerID, loggingInterval, &varArray); // ========================================================================== /** Start [publishers] */ // Create a data publisher for the Monitor My Watershed/EnviroDIY POST endpoint -#include -EnviroDIYPublisher EnviroDIYPOST(dataLogger, &modem.gsmClient, +#include +EventHubPublisher EnviroDIYPOST(dataLogger, &modem.gsmClient, registrationToken, samplingFeature); /** End [publishers] */ diff --git a/src/publishers/EventHubPublisher.cpp b/src/publishers/EventHubPublisher.cpp index ddc5e714b..7126adeae 100644 --- a/src/publishers/EventHubPublisher.cpp +++ b/src/publishers/EventHubPublisher.cpp @@ -12,16 +12,16 @@ // ============================================================================ -// Functions for the EnviroDIY data portal receivers. +// Functions for the Azure Event Hub REST API. // ============================================================================ // Constant values for post requests // I want to refer to these more than once while ensuring there is only one copy // in memory -const char* EventHubPublisher::postEndpoint = "/api/data-stream/"; -const char* EventHubPublisher::enviroDIYHost = "data.envirodiy.org"; -const int EventHubPublisher::enviroDIYPort = 80; -const char* EventHubPublisher::tokenHeader = "\r\nTOKEN: "; +const char* EventHubPublisher::postEndpoint = "/devices/messages"; +const char* EventHubPublisher::eventHubHost = "event-hub-data-logger.servicebus.windows.net"; +const int EventHubPublisher::eventHubPort = 80; +const char* EventHubPublisher::tokenHeader = "\r\nAuthorization: "; // const unsigned char *EventHubPublisher::cacheHeader = "\r\nCache-Control: // no-cache"; const unsigned char *EventHubPublisher::connectionHeader = // "\r\nConnection: close"; @@ -29,7 +29,7 @@ const char* EventHubPublisher::contentLengthHeader = "\r\nContent-Length: "; const char* EventHubPublisher::contentTypeHeader = "\r\nContent-Type: application/json\r\n\r\n"; -const char* EventHubPublisher::samplingFeatureTag = "{\"sampling_feature\":\""; +const char* EventHubPublisher::samplingFeatureTag = "{\"id\":\""; const char* EventHubPublisher::timestampTag = "\",\"timestamp\":\""; @@ -79,14 +79,14 @@ void EventHubPublisher::setToken(const char* registrationToken) { // Calculates how long the JSON will be uint16_t EventHubPublisher::calculateJsonSize() { - uint16_t jsonLength = 21; // {"sampling_feature":" + uint16_t jsonLength = 7; // {"id":" jsonLength += 36; // sampling feature UUID jsonLength += 15; // ","timestamp":" jsonLength += 25; // markedISO8601Time jsonLength += 2; // ", for (uint8_t i = 0; i < _baseLogger->getArrayVarCount(); i++) { jsonLength += 1; // " - jsonLength += 36; // variable UUID + jsonLength += 11; // variable code (i.e. "Mayfly_Temp") jsonLength += 2; // ": jsonLength += _baseLogger->getValueStringAtI(i).length(); if (i + 1 != _baseLogger->getArrayVarCount()) { @@ -118,7 +118,7 @@ uint16_t EventHubPublisher::calculatePostSize() */ -// This prints a properly formatted JSON for EnviroDIY to an Arduino stream +// This prints a properly formatted JSON for EventHub to an Arduino stream void EventHubPublisher::printSensorDataJSON(Stream* stream) { stream->print(samplingFeatureTag); stream->print(_baseLogger->getSamplingFeatureUUID()); @@ -139,15 +139,15 @@ void EventHubPublisher::printSensorDataJSON(Stream* stream) { } -// This prints a fully structured post request for EnviroDIY to the +// This prints a fully structured post request for EventHub to the // specified stream. -void EventHubPublisher::printEnviroDIYRequest(Stream* stream) { +void EventHubPublisher::printEventHubRequest(Stream* stream) { // Stream the HTTP headers for the post request stream->print(postHeader); stream->print(postEndpoint); stream->print(HTTPtag); stream->print(hostHeader); - stream->print(enviroDIYHost); + stream->print(eventHubHost); stream->print(tokenHeader); stream->print(_registrationToken); // stream->print(cacheHeader); @@ -179,10 +179,10 @@ void EventHubPublisher::begin(Logger& baseLogger, // This utilizes an attached modem to make a TCP connection to the -// EnviroDIY/ODM2DataSharingPortal and then streams out a post request +// Azure EventHub and then streams out a post request // over that connection. // The return is the http status code of the response. -// int16_t EventHubPublisher::postDataEnviroDIY(void) +// int16_t EnviroDIYPublisher::postDataEnviroDIY(void) int16_t EventHubPublisher::publishData(Client* outClient) { // Create a buffer for the portions of the request and response char tempBuffer[37] = ""; @@ -193,7 +193,7 @@ int16_t EventHubPublisher::publishData(Client* outClient) { // Open a TCP/IP connection to the Enviro DIY Data Portal (WebSDL) MS_DBG(F("Connecting client")); MS_START_DEBUG_TIMER; - if (outClient->connect(enviroDIYHost, enviroDIYPort)) { + if (outClient->connect(eventHubHost, eventHubPort)) { MS_DBG(F("Client connected after"), MS_PRINT_DEBUG_TIMER, F("ms\n")); // copy the initial post header into the tx buffer @@ -206,7 +206,7 @@ int16_t EventHubPublisher::publishData(Client* outClient) { // there is space for that line, sending out buffer if not if (bufferFree() < 28) printTxBuffer(outClient); strcat(txBuffer, hostHeader); - strcat(txBuffer, enviroDIYHost); + strcat(txBuffer, eventHubHost); if (bufferFree() < 47) printTxBuffer(outClient); strcat(txBuffer, tokenHeader); @@ -279,7 +279,7 @@ int16_t EventHubPublisher::publishData(Client* outClient) { outClient->stop(); MS_DBG(F("Client stopped after"), MS_PRINT_DEBUG_TIMER, F("ms")); } else { - PRINTOUT(F("\n -- Unable to Establish Connection to EnviroDIY Data " + PRINTOUT(F("\n -- Unable to Establish Connection to EventHub REST API " "Portal --")); } diff --git a/src/publishers/EventHubPublisher.h b/src/publishers/EventHubPublisher.h index 8120eaaf5..3f156055c 100644 --- a/src/publishers/EventHubPublisher.h +++ b/src/publishers/EventHubPublisher.h @@ -6,8 +6,8 @@ * @author Anthony Aufdenkampe * * @brief Contains the EventHubPublisher subclass of dataPublisher for - * publishing data to the Monitor My Watershed/EnviroDIY data portal at - * http://data.enviroDIY.org + * publishing data to Azure Event Hub REST API + * https://docs.microsoft.com/en-us/rest/api/eventhub/event-hubs-runtime-rest */ // Header Guards @@ -28,12 +28,12 @@ // ============================================================================ -// Functions for the EnviroDIY data portal receivers. +// Functions for the Azure Event Hub REST API. // ============================================================================ /** * @brief The EventHubPublisher subclass of dataPublisher for publishing data - * to the Monitor My Watershed/EnviroDIY data portal at - * http://data.enviroDIY.org + * publishing data to Azure Event Hub REST API + * https://docs.microsoft.com/en-us/rest/api/eventhub/event-hubs-runtime-rest * * @ingroup the_publishers */ @@ -132,7 +132,7 @@ class EventHubPublisher : public dataPublisher { // Returns the data destination String getEndpoint(void) override { - return String(enviroDIYHost); + return String(eventHubHost); } // Adds the site registration token @@ -173,7 +173,7 @@ class EventHubPublisher : public dataPublisher { * * @param stream The Arduino stream to write out the request to. */ - void printEnviroDIYRequest(Stream* stream); + void printEventHubRequest(Stream* stream); // A way to begin with everything already set /** @@ -219,8 +219,8 @@ class EventHubPublisher : public dataPublisher { * @{ */ static const char* postEndpoint; ///< The endpoint - static const char* enviroDIYHost; ///< The host name - static const int enviroDIYPort; ///< The host port + static const char* eventHubHost; ///< The host name + static const int eventHubPort; ///< The host port static const char* tokenHeader; ///< The token header text // static const char *cacheHeader; ///< The cache header text // static const char *connectionHeader; ///< The keep alive header text @@ -243,4 +243,4 @@ class EventHubPublisher : public dataPublisher { const char* _registrationToken; }; -#endif // SRC_PUBLISHERS_EventHubPublisher_H_ +#endif // SRC_PUBLISHERS_EVENTHUBPUBLISHER_H_ From 03579d5a54cfad2c3d7cd019219c20c886068fe5 Mon Sep 17 00:00:00 2001 From: Anthony Aufdenkampe Date: Fri, 15 Apr 2022 17:47:54 -0500 Subject: [PATCH 4/8] EventHubPublisher connecting; not quite there Set Port to 443 for HTTPS, and now connecting! Still getting 504. After using postman, I had thought that the issue was related to Content-Length being wrong, but then I would have gotten a 411 response code. Tried Transfer-Encoding: chunked, but still 504 Tried adding [] around json, but still 504. --- .../HMACEventHubTest/HMACEventHubTest.ino | 2 +- src/publishers/EventHubPublisher.cpp | 35 ++++++++++++------- src/publishers/EventHubPublisher.h | 1 + 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino b/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino index a094f3989..8332b63bd 100644 --- a/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino +++ b/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino @@ -172,7 +172,7 @@ const char* UUIDs[] = // UUID array for device sensors { // formatted below to all have 11 characters // "Mayfly_Batt", // Battery voltage (EnviroDIY_Mayfly_Batt) - "Mayfly_Temp", // Board Temperature (EnviroDIY_Mayfly_Temp) + "measurement", // Board Temperature (EnviroDIY_Mayfly_Temp) // "LTEB_Signal", // Percent full scale (EnviroDIY_LTEB_SignalPercent) }; const char* registrationToken = "SharedAccessSignature sr=https%3A%2F%2Fevent-hub-data-logger.servicebus.windows.net%2Fdevices%2Fmessages&sig=fHhAkb/uw/8W2DcVMYJePm0K4QVabKAv2CRE9j3Al1A%3D&se=1650074784&skn=mayfly-device"; // Device registration token diff --git a/src/publishers/EventHubPublisher.cpp b/src/publishers/EventHubPublisher.cpp index 7126adeae..891965c7d 100644 --- a/src/publishers/EventHubPublisher.cpp +++ b/src/publishers/EventHubPublisher.cpp @@ -18,18 +18,21 @@ // Constant values for post requests // I want to refer to these more than once while ensuring there is only one copy // in memory -const char* EventHubPublisher::postEndpoint = "/devices/messages"; +const char* EventHubPublisher::postEndpoint = "https://event-hub-data-logger.servicebus.windows.net/devices/messages"; const char* EventHubPublisher::eventHubHost = "event-hub-data-logger.servicebus.windows.net"; -const int EventHubPublisher::eventHubPort = 80; +const int EventHubPublisher::eventHubPort = 443; // 443 for HTTPS; 80 for HTTP const char* EventHubPublisher::tokenHeader = "\r\nAuthorization: "; -// const unsigned char *EventHubPublisher::cacheHeader = "\r\nCache-Control: -// no-cache"; const unsigned char *EventHubPublisher::connectionHeader = -// "\r\nConnection: close"; +// const unsigned char *EventHubPublisher::cacheHeader = +// "\r\nCache-Control: no-cache"; +// const unsigned char *EventHubPublisher::connectionHeader = +// "\r\nConnection: close"; +const char* EventHubPublisher::transferEncodingHeader = + "\r\nTransfer-Encoding: chunked"; const char* EventHubPublisher::contentLengthHeader = "\r\nContent-Length: "; const char* EventHubPublisher::contentTypeHeader = - "\r\nContent-Type: application/json\r\n\r\n"; + "\r\nContent-Type: application/json; charset=utf-8\r\n\r\n"; -const char* EventHubPublisher::samplingFeatureTag = "{\"id\":\""; +const char* EventHubPublisher::samplingFeatureTag = "[{\"id\":\""; const char* EventHubPublisher::timestampTag = "\",\"timestamp\":\""; @@ -86,7 +89,7 @@ uint16_t EventHubPublisher::calculateJsonSize() { jsonLength += 2; // ", for (uint8_t i = 0; i < _baseLogger->getArrayVarCount(); i++) { jsonLength += 1; // " - jsonLength += 11; // variable code (i.e. "Mayfly_Temp") + jsonLength += 11; // variable code (i.e. "measurement") jsonLength += 2; // ": jsonLength += _baseLogger->getValueStringAtI(i).length(); if (i + 1 != _baseLogger->getArrayVarCount()) { @@ -136,6 +139,7 @@ void EventHubPublisher::printSensorDataJSON(Stream* stream) { } stream->print('}'); + stream->print(']'); } @@ -152,8 +156,9 @@ void EventHubPublisher::printEventHubRequest(Stream* stream) { stream->print(_registrationToken); // stream->print(cacheHeader); // stream->print(connectionHeader); - stream->print(contentLengthHeader); - stream->print(calculateJsonSize()); + stream->print(transferEncodingHeader); + // stream->print(contentLengthHeader); + // stream->print(calculateJsonSize()); stream->print(contentTypeHeader); // Stream the JSON itself @@ -219,9 +224,12 @@ int16_t EventHubPublisher::publishData(Client* outClient) { // strcat(txBuffer, connectionHeader); if (bufferFree() < 26) printTxBuffer(outClient); - strcat(txBuffer, contentLengthHeader); - itoa(calculateJsonSize(), tempBuffer, 10); // BASE 10 - strcat(txBuffer, tempBuffer); + strcat(txBuffer, transferEncodingHeader); + + // if (bufferFree() < 26) printTxBuffer(outClient); + // strcat(txBuffer, contentLengthHeader); + // itoa(calculateJsonSize(), tempBuffer, 10); // BASE 10 + // strcat(txBuffer, tempBuffer); if (bufferFree() < 42) printTxBuffer(outClient); strcat(txBuffer, contentTypeHeader); @@ -256,6 +264,7 @@ int16_t EventHubPublisher::publishData(Client* outClient) { txBuffer[strlen(txBuffer)] = ','; } else { txBuffer[strlen(txBuffer)] = '}'; + txBuffer[strlen(txBuffer)] = ']'; } } diff --git a/src/publishers/EventHubPublisher.h b/src/publishers/EventHubPublisher.h index 3f156055c..b5a3ddcb6 100644 --- a/src/publishers/EventHubPublisher.h +++ b/src/publishers/EventHubPublisher.h @@ -224,6 +224,7 @@ class EventHubPublisher : public dataPublisher { static const char* tokenHeader; ///< The token header text // static const char *cacheHeader; ///< The cache header text // static const char *connectionHeader; ///< The keep alive header text + static const char* transferEncodingHeader; ///< chunked? static const char* contentLengthHeader; ///< The content length header text static const char* contentTypeHeader; ///< The content type header text /**@}*/ From f9cfc0864316ff909973c21ee0ffbafc421231b8 Mon Sep 17 00:00:00 2001 From: Anthony Aufdenkampe Date: Mon, 18 Apr 2022 16:52:40 -0500 Subject: [PATCH 5/8] Event Hub cleanup & debugging; but not working! I still can not get this to POST to Event Hub, even with a properly formatted POST Request (that works in Postman). It connects to the server, but gets no response even after 60 seconds. @SRGDamia1, can you help me figure out #441? --- .../HMACEventHubTest/HMACEventHubTest.ino | 2 +- src/publishers/EnviroDIYPublisher.cpp | 7 ++- src/publishers/EventHubPublisher.cpp | 57 ++++++++++--------- src/publishers/EventHubPublisher.h | 33 ++++++----- 4 files changed, 51 insertions(+), 48 deletions(-) diff --git a/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino b/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino index 8332b63bd..f8b3b7eff 100644 --- a/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino +++ b/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino @@ -175,7 +175,7 @@ const char* UUIDs[] = // UUID array for device sensors "measurement", // Board Temperature (EnviroDIY_Mayfly_Temp) // "LTEB_Signal", // Percent full scale (EnviroDIY_LTEB_SignalPercent) }; -const char* registrationToken = "SharedAccessSignature sr=https%3A%2F%2Fevent-hub-data-logger.servicebus.windows.net%2Fdevices%2Fmessages&sig=fHhAkb/uw/8W2DcVMYJePm0K4QVabKAv2CRE9j3Al1A%3D&se=1650074784&skn=mayfly-device"; // Device registration token +const char* registrationToken = "SharedAccessSignature sr=https%3A%2F%2Fevent-hub-data-logger.servicebus.windows.net%2Fdevices%2Fmessages&sig=pDocq7bRcZpKgt%2BWmNQHsJjz36mcssxF0EQ6jnVjd/g%3D&se=1650317059&skn=mayfly-device"; // Device registration token const char* samplingFeature = "7d37e135-0e26-4bc7-aa81-f9443283582d"; // Sampling feature UUID diff --git a/src/publishers/EnviroDIYPublisher.cpp b/src/publishers/EnviroDIYPublisher.cpp index d75845d05..6a2f64546 100644 --- a/src/publishers/EnviroDIYPublisher.cpp +++ b/src/publishers/EnviroDIYPublisher.cpp @@ -21,9 +21,10 @@ const char* EnviroDIYPublisher::postEndpoint = "/api/data-stream/"; const char* EnviroDIYPublisher::enviroDIYHost = "data.envirodiy.org"; const int EnviroDIYPublisher::enviroDIYPort = 80; const char* EnviroDIYPublisher::tokenHeader = "\r\nTOKEN: "; -// const unsigned char *EnviroDIYPublisher::cacheHeader = "\r\nCache-Control: -// no-cache"; const unsigned char *EnviroDIYPublisher::connectionHeader = -// "\r\nConnection: close"; +// const char* EnviroDIYPublisher::cacheHeader = +// "\r\nCache-Control: no-cache"; +// const char* EnviroDIYPublisher::connectionHeader = +// "\r\nConnection: close"; const char* EnviroDIYPublisher::contentLengthHeader = "\r\nContent-Length: "; const char* EnviroDIYPublisher::contentTypeHeader = "\r\nContent-Type: application/json\r\n\r\n"; diff --git a/src/publishers/EventHubPublisher.cpp b/src/publishers/EventHubPublisher.cpp index 891965c7d..4b584926f 100644 --- a/src/publishers/EventHubPublisher.cpp +++ b/src/publishers/EventHubPublisher.cpp @@ -18,21 +18,21 @@ // Constant values for post requests // I want to refer to these more than once while ensuring there is only one copy // in memory -const char* EventHubPublisher::postEndpoint = "https://event-hub-data-logger.servicebus.windows.net/devices/messages"; +const char* EventHubPublisher::postEndpoint = "https://event-hub-data-logger.servicebus.windows.net/devices/messages?timeout=60"; const char* EventHubPublisher::eventHubHost = "event-hub-data-logger.servicebus.windows.net"; const int EventHubPublisher::eventHubPort = 443; // 443 for HTTPS; 80 for HTTP const char* EventHubPublisher::tokenHeader = "\r\nAuthorization: "; -// const unsigned char *EventHubPublisher::cacheHeader = +// const char* EventHubPublisher::cacheHeader = // "\r\nCache-Control: no-cache"; -// const unsigned char *EventHubPublisher::connectionHeader = +// const char* EventHubPublisher::connectionHeader = // "\r\nConnection: close"; -const char* EventHubPublisher::transferEncodingHeader = - "\r\nTransfer-Encoding: chunked"; +// const char* EventHubPublisher::transferEncodingHeader = +// "\r\nTransfer-Encoding: chunked"; const char* EventHubPublisher::contentLengthHeader = "\r\nContent-Length: "; const char* EventHubPublisher::contentTypeHeader = "\r\nContent-Type: application/json; charset=utf-8\r\n\r\n"; -const char* EventHubPublisher::samplingFeatureTag = "[{\"id\":\""; +const char* EventHubPublisher::samplingFeatureTag = "{\"id\":\""; const char* EventHubPublisher::timestampTag = "\",\"timestamp\":\""; @@ -107,7 +107,7 @@ uint16_t EventHubPublisher::calculateJsonSize() { uint16_t EventHubPublisher::calculatePostSize() { uint16_t postLength = 31; // "POST /api/data-stream/ HTTP/1.1" - postLength += 28; // "\r\nHost: data.envirodiy.org" + postLength += 28; // "\r\nHost: event-hub-data-logger.servicebus.windows.net" postLength += 11; // "\r\nTOKEN: " postLength += 36; // registrationToken // postLength += 27; // "\r\nCache-Control: no-cache" @@ -139,7 +139,6 @@ void EventHubPublisher::printSensorDataJSON(Stream* stream) { } stream->print('}'); - stream->print(']'); } @@ -156,9 +155,9 @@ void EventHubPublisher::printEventHubRequest(Stream* stream) { stream->print(_registrationToken); // stream->print(cacheHeader); // stream->print(connectionHeader); - stream->print(transferEncodingHeader); - // stream->print(contentLengthHeader); - // stream->print(calculateJsonSize()); + // stream->print(transferEncodingHeader); + stream->print(contentLengthHeader); + stream->print(calculateJsonSize()); stream->print(contentTypeHeader); // Stream the JSON itself @@ -187,10 +186,10 @@ void EventHubPublisher::begin(Logger& baseLogger, // Azure EventHub and then streams out a post request // over that connection. // The return is the http status code of the response. -// int16_t EnviroDIYPublisher::postDataEnviroDIY(void) int16_t EventHubPublisher::publishData(Client* outClient) { // Create a buffer for the portions of the request and response char tempBuffer[37] = ""; + char respondBuffer[500] = ""; uint16_t did_respond = 0; MS_DBG(F("Outgoing JSON size:"), calculateJsonSize()); @@ -223,13 +222,13 @@ int16_t EventHubPublisher::publishData(Client* outClient) { // if (bufferFree() < 21) printTxBuffer(outClient); // strcat(txBuffer, connectionHeader); - if (bufferFree() < 26) printTxBuffer(outClient); - strcat(txBuffer, transferEncodingHeader); - // if (bufferFree() < 26) printTxBuffer(outClient); - // strcat(txBuffer, contentLengthHeader); - // itoa(calculateJsonSize(), tempBuffer, 10); // BASE 10 - // strcat(txBuffer, tempBuffer); + // strcat(txBuffer, transferEncodingHeader); + + if (bufferFree() < 26) printTxBuffer(outClient); + strcat(txBuffer, contentLengthHeader); + itoa(calculateJsonSize(), tempBuffer, 10); // BASE 10 + strcat(txBuffer, tempBuffer); if (bufferFree() < 42) printTxBuffer(outClient); strcat(txBuffer, contentTypeHeader); @@ -264,23 +263,24 @@ int16_t EventHubPublisher::publishData(Client* outClient) { txBuffer[strlen(txBuffer)] = ','; } else { txBuffer[strlen(txBuffer)] = '}'; - txBuffer[strlen(txBuffer)] = ']'; } } // Send out the finished request (or the last unsent section of it) printTxBuffer(outClient, true); - // Wait 10 seconds for a response from the server + // Wait 10 seconds for a response from the server, up to 500 characters + MS_DBG(F("Waiting for response from server")); uint32_t start = millis(); - while ((millis() - start) < 10000L && outClient->available() < 12) { - delay(10); + while ((millis() - start) < 10000L && outClient->available() < 500) { + delay(100); + Serial.print(F(".")); } - // Read only the first 12 characters of the response + // Read only the first 500 characters of the response // We're only reading as far as the http code, anything beyond that // we don't care about. - did_respond = outClient->readBytes(tempBuffer, 12); + did_respond = outClient->readBytes(respondBuffer, 500); // Close the TCP/IP connection MS_DBG(F("Stopping client")); @@ -292,19 +292,22 @@ int16_t EventHubPublisher::publishData(Client* outClient) { "Portal --")); } - // Process the HTTP response + // Print entire response + MS_DBG(F("\n-- Response Header & Body --\n"),did_respond, respondBuffer); + + // Process the HTTP response code int16_t responseCode = 0; if (did_respond > 0) { char responseCode_char[4]; for (uint8_t i = 0; i < 3; i++) { - responseCode_char[i] = tempBuffer[i + 9]; + responseCode_char[i] = respondBuffer[i + 9]; } responseCode = atoi(responseCode_char); } else { responseCode = 504; } - PRINTOUT(F("-- Response Code --")); + PRINTOUT(F("\n-- Response Code --")); PRINTOUT(responseCode); return responseCode; diff --git a/src/publishers/EventHubPublisher.h b/src/publishers/EventHubPublisher.h index b5a3ddcb6..89173407c 100644 --- a/src/publishers/EventHubPublisher.h +++ b/src/publishers/EventHubPublisher.h @@ -41,11 +41,11 @@ class EventHubPublisher : public dataPublisher { public: // Constructors /** - * @brief Construct a new EnviroDIY Publisher object with no members set. + * @brief Construct a new Event Hub REST API Publisher object with no members set. */ EventHubPublisher(); /** - * @brief Construct a new EnviroDIY Publisher object + * @brief Construct a new Event Hub REST API Publisher object * * @note If a client is never specified, the publisher will attempt to * create and use a client on a LoggerModem instance tied to the attached @@ -67,7 +67,7 @@ class EventHubPublisher : public dataPublisher { explicit EventHubPublisher(Logger& baseLogger, uint8_t sendEveryX = 1, uint8_t sendOffset = 0); /** - * @brief Construct a new EnviroDIY Publisher object + * @brief Construct a new Event Hub REST API Publisher object * * @param baseLogger The logger supplying the data to be published * @param inClient An Arduino client instance to use to print data to. @@ -88,7 +88,7 @@ class EventHubPublisher : public dataPublisher { EventHubPublisher(Logger& baseLogger, Client* inClient, uint8_t sendEveryX = 1, uint8_t sendOffset = 0); /** - * @brief Construct a new EnviroDIY Publisher object + * @brief Construct a new Event Hub REST API Publisher object * * @param baseLogger The logger supplying the data to be published * @param registrationToken The registration token for the site on the @@ -105,7 +105,7 @@ class EventHubPublisher : public dataPublisher { const char* samplingFeatureUUID, uint8_t sendEveryX = 1, uint8_t sendOffset = 0); /** - * @brief Construct a new EnviroDIY Publisher object + * @brief Construct a new Event Hub REST API Publisher object * * @param baseLogger The logger supplying the data to be published * @param inClient An Arduino client instance to use to print data to. @@ -126,7 +126,7 @@ class EventHubPublisher : public dataPublisher { const char* samplingFeatureUUID, uint8_t sendEveryX = 1, uint8_t sendOffset = 0); /** - * @brief Destroy the EnviroDIY Publisher object + * @brief Destroy the Event Hub REST API Publisher object */ virtual ~EventHubPublisher(); @@ -160,7 +160,7 @@ class EventHubPublisher : public dataPublisher { // uint16_t calculatePostSize(); /** - * @brief This generates a properly formatted JSON for EnviroDIY and prints + * @brief This generates a properly formatted JSON for Event Hub and prints * it to the input Arduino stream object. * * @param stream The Arduino stream to write out the JSON to. @@ -168,8 +168,8 @@ class EventHubPublisher : public dataPublisher { void printSensorDataJSON(Stream* stream); /** - * @brief This prints a fully structured post request for Monitor My - * Watershed/EnviroDIY to the specified stream. + * @brief This prints a fully structured post request for Azure Event Hub + * to the specified stream. * * @param stream The Arduino stream to write out the request to. */ @@ -195,10 +195,9 @@ class EventHubPublisher : public dataPublisher { void begin(Logger& baseLogger, const char* registrationToken, const char* samplingFeatureUUID); - // int16_t postDataEnviroDIY(void); /** * @brief Utilize an attached modem to open a a TCP connection to the - * EnviroDIY/ODM2DataSharingPortal and then stream out a post request over + * Azure Event Hub and then stream out a post request over * that connection. * * This depends on an internet connection already having been made and a @@ -213,8 +212,8 @@ class EventHubPublisher : public dataPublisher { protected: /** - * @anchor envirodiy_post_vars - * @name Portions of the POST request to EnviroDIY + * @anchor eventhub_post_vars + * @name Portions of the POST request to Event Hub * * @{ */ @@ -222,7 +221,7 @@ class EventHubPublisher : public dataPublisher { static const char* eventHubHost; ///< The host name static const int eventHubPort; ///< The host port static const char* tokenHeader; ///< The token header text - // static const char *cacheHeader; ///< The cache header text + static const char* cacheHeader; ///< The cache header text // static const char *connectionHeader; ///< The keep alive header text static const char* transferEncodingHeader; ///< chunked? static const char* contentLengthHeader; ///< The content length header text @@ -230,8 +229,8 @@ class EventHubPublisher : public dataPublisher { /**@}*/ /** - * @anchor envirodiy_json_vars - * @name Portions of the JSON object for EnviroDIY + * @anchor eventhub_json_vars + * @name Portions of the JSON object for Event Hub * * @{ */ @@ -240,7 +239,7 @@ class EventHubPublisher : public dataPublisher { /**@}*/ private: - // Tokens and UUID's for EnviroDIY + // Tokens and UUID's for Event Hub const char* _registrationToken; }; From 4b624c7b4ef3783efdd1ae01e200c1eecfecd606 Mon Sep 17 00:00:00 2001 From: Anthony Aufdenkampe Date: Tue, 19 Apr 2022 09:19:13 -0500 Subject: [PATCH 6/8] Add EventHub test w/ Digi radio; still not working @GArrigotti-cws, @GArrigotti, I also added `platformio.ini` files that you can use for testing on your side. Connected to #411 --- examples/DRWI_DigiLTE/DRWI_DigiLTE.ino | 1 + sensor_tests/HMACEventHubTest/ReadMe.md | 44 -- sensor_tests/HMACEventHubTest/platformio.ini | 20 +- .../HMACEventHubTest_Digi.ino | 396 ++++++++++++++++++ .../HMACEventHubTest_Digi/platformio.ini | 41 ++ src/publishers/EventHubPublisher.cpp | 3 +- 6 files changed, 453 insertions(+), 52 deletions(-) delete mode 100644 sensor_tests/HMACEventHubTest/ReadMe.md create mode 100644 sensor_tests/HMACEventHubTest_Digi/HMACEventHubTest_Digi.ino create mode 100644 sensor_tests/HMACEventHubTest_Digi/platformio.ini diff --git a/examples/DRWI_DigiLTE/DRWI_DigiLTE.ino b/examples/DRWI_DigiLTE/DRWI_DigiLTE.ino index 19e9e7ba6..20682c6a6 100644 --- a/examples/DRWI_DigiLTE/DRWI_DigiLTE.ino +++ b/examples/DRWI_DigiLTE/DRWI_DigiLTE.ino @@ -3,6 +3,7 @@ * @brief Example for DRWI CitSci LTE sites. * * @author Sara Geleskie Damiano + * @author Anthony Aufdenkampe * @copyright (c) 2017-2022 Stroud Water Research Center (SWRC) * and the EnviroDIY Development Team * This example is published under the BSD-3 license. diff --git a/sensor_tests/HMACEventHubTest/ReadMe.md b/sensor_tests/HMACEventHubTest/ReadMe.md deleted file mode 100644 index 0f9e164c2..000000000 --- a/sensor_tests/HMACEventHubTest/ReadMe.md +++ /dev/null @@ -1,44 +0,0 @@ -# DRWI Sites with EnviroDIY LTE Bees -Example sketch for using the EnviroDIY SIM7080G LTE cellular module with an EnviroDIY Mayfly Data Logger. - -The exact hardware configuration used in this example: - * Mayfly v1.0 board - * EnviroDIY SIM7080 LTE module (with Hologram SIM card) - * Hydros21 CTD sensor - * Campbell Scientific OBS3+ Turbidity sensor - -An EnviroDIY LTE SIM7080 module can be used with the older Mayfly v0.5b boards if you change line 101 (for modemVccPin) from 18 to -1. -This is because the Mayfly v1.0 board has a separate 3.3v regulator to power the Bee socket and is controlled by turning pin 18 on or off. -Mayfly v0.5b has the Bee socket constantly powered, therefore using "-1" is the proper setting for that line of code. - -The EnviroDIY LTE SIM7080 module includes 2 antennas in the package. The small thin one is the cellular antenna, and should be connected to the socket labeled "CELL". The thicker block is the GPS antenna, and should be connected to the "GPS" socket, but only if you intend to use the GPS functionality of the module. ModularSensors does not currently suport GPS functionality, but other libraries such as TinyGPS can work with the SIM7080 module. - -The included cell antenna works best in high-signal-strength areas. For most remote areas and logger deployments, we suggest a larger LTE antenna, like the W3907B0100 -from PulseLarsen (Digikey 1837-1003-ND or Mouser 673-W3907B0100) - -_______ - -[//]: # ( @tableofcontents ) - -[//]: # ( @m_footernavigation ) - -[//]: # ( Start GitHub Only ) -- [DRWI Sites with EnviroDIY LTE Bees](#drwi-sites-with-envirodiy-lte-bees) -- [Unique Features of the DRWI EnviroDIY LTE Example](#unique-features-of-the-drwi-envirodiy-lte-example) - -[//]: # ( End GitHub Only ) - -_______ - -# Unique Features of the DRWI EnviroDIY LTE Example -- Specifically for sites within the Delaware River Watershed Initiative. -- Uses a EnviroDIY LTE Bee based on the SIMCom SIM7080G - - -[//]: # ( @section example_drwi_ediylte_pio_config PlatformIO Configuration ) - -[//]: # ( @include{lineno} DRWI_SIM7080LTE/platformio.ini ) - -[//]: # ( @section example_drwi_ediylte_code The Complete Code ) - -[//]: # ( @include{lineno} DRWI_SIM7080LTE/DRWI_SIM7080LTE.ino ) diff --git a/sensor_tests/HMACEventHubTest/platformio.ini b/sensor_tests/HMACEventHubTest/platformio.ini index 8acac309a..2c83c968b 100644 --- a/sensor_tests/HMACEventHubTest/platformio.ini +++ b/sensor_tests/HMACEventHubTest/platformio.ini @@ -9,10 +9,11 @@ ; http://docs.platformio.org/page/projectconf.html [platformio] -description = ModularSensors example intended for DRWI users with CTD, turbidity, and a EnviroDIY SIM7080G LTE modem +description = Test HMAC & POST Requests to Event Hub API, using LTEBee (SIM27080) radio & Mayfly 1.1 +src_dir = sensor_tests/HMACEventHubTest [env:mayfly] -monitor_speed = 57600 +monitor_speed = 115200 board = mayfly platform = atmelavr framework = arduino @@ -27,9 +28,14 @@ lib_ignore = Adafruit TouchScreen Adafruit ILI9341 build_flags = - -DSDI12_EXTERNAL_PCINT + -D SDI12_EXTERNAL_PCINT + -D TINY_GSM_RX_BUFFER=512 + -D TINY_GSM_YIELD_MS=2 + -D MS_DATAPUBLISHERBASE_DEBUG + -D MS_EVENTHUBPUBLISHER_DEBUG + ; -D TINY_GSM_DEBUG=Serial + ; -D MS_LOGGERMODEM_DEBUG + ; -D MS_DIGIXBEECELLULARTRANSPARENT_DEBUG lib_deps = - envirodiy/EnviroDIY_ModularSensors -; ^^ Use this when working from an official release of the library -; https://github.com/EnviroDIY/ModularSensors.git#develop -; ^^ Use this when if you want to pull from the develop branch + https://github.com/EnviroDIY/ModularSensors.git#hmac_auth +; ^^ Use this when if you want to pull from the hmac_auth feature branch diff --git a/sensor_tests/HMACEventHubTest_Digi/HMACEventHubTest_Digi.ino b/sensor_tests/HMACEventHubTest_Digi/HMACEventHubTest_Digi.ino new file mode 100644 index 000000000..d98bb00d5 --- /dev/null +++ b/sensor_tests/HMACEventHubTest_Digi/HMACEventHubTest_Digi.ino @@ -0,0 +1,396 @@ +/** ========================================================================= + * @file HMACEventHubTest_Digi.ino + * @brief Test, based off DRWI_DigiLTE.ino.ino + * + * This example shows proper settings for the following configuration: + * + * Mayfly v1.1 board + * EnviroDIY SIM7080 LTE module (with Hologram SIM card) + * + * @author Sara Geleskie Damiano + * @author Anthony Aufdenkampe + * @copyright (c) 2017-2022 Stroud Water Research Center (SWRC) + * and the EnviroDIY Development Team + * This example is published under the BSD-3 license. + * + * Build Environment: Visual Studios Code with PlatformIO + * Hardware Platform: EnviroDIY Mayfly Arduino Datalogger + * + * DISCLAIMER: + * THIS CODE IS PROVIDED "AS IS" - NO WARRANTY IS GIVEN. + * ======================================================================= */ + +// ========================================================================== +// Defines for the Arduino IDE +// NOTE: These are ONLY needed to compile with the Arduino IDE. +// If you use PlatformIO, you should set these build flags in your +// platformio.ini +// ========================================================================== +/** Start [defines] */ +#ifndef TINY_GSM_RX_BUFFER +#define TINY_GSM_RX_BUFFER 64 +#endif +#ifndef TINY_GSM_YIELD_MS +#define TINY_GSM_YIELD_MS 2 +#endif +/** End [defines] */ + +// ========================================================================== +// Include the libraries required for any data logger +// ========================================================================== +/** Start [includes] */ +// The Arduino library is needed for every Arduino program. +#include + +// EnableInterrupt is used by ModularSensors for external and pin change +// interrupts and must be explicitly included in the main program. +#include + +// Include the main header for ModularSensors +#include +/** End [includes] */ + + +// ========================================================================== +// Data Logging Options +// ========================================================================== +/** Start [logging_options] */ +// The name of this program file +const char* sketchName = "HMACEventHubTest_Digi.ino"; +// Logger ID, also becomes the prefix for the name of the data file on SD card +const char* LoggerID = "HMACEventHubTest_Digi"; +// How frequently (in minutes) to log data +const uint8_t loggingInterval = 1; +// Your logger's timezone. +const int8_t timeZone = -5; // Eastern Standard Time +// NOTE: Daylight savings time will not be applied! Please use standard time! + +// Set the input and output pins for the logger +// NOTE: Use -1 for pins that do not apply +const int32_t serialBaud = 115200; // Baud rate for debugging +const int8_t greenLED = 8; // Pin for the green LED +const int8_t redLED = 9; // Pin for the red LED +const int8_t buttonPin = 21; // Pin for debugging mode (ie, button pin) +const int8_t wakePin = 31; // MCU interrupt/alarm pin to wake from sleep +// Mayfly 0.x D31 = A7 +const int8_t sdCardPwrPin = -1; // MCU SD card power pin +const int8_t sdCardSSPin = 12; // SD card chip select/slave select pin +const int8_t sensorPowerPin = 22; // MCU pin controlling main sensor power +/** End [logging_options] */ + + +// ========================================================================== +// Wifi/Cellular Modem Options +// ========================================================================== +/** Start [digi_xbee_cellular_transparent] */ +// For any Digi Cellular XBee's +// NOTE: The u-blox based Digi XBee's (3G global and LTE-M global) +// are more stable used in bypass mode (below) +// The Telit based Digi XBees (LTE Cat1) can only use this mode. +#include + +// Create a reference to the serial port for the modem +HardwareSerial& modemSerial = Serial1; // Use hardware serial if possible +const int32_t modemBaud = 9600; // All XBee's use 9600 by default + +// Modem Pins - Describe the physical pin connection of your modem to your board +// NOTE: Use -1 for pins that do not apply +const int8_t modemVccPin = -2; // MCU pin controlling modem power +const int8_t modemStatusPin = 19; // MCU pin used to read modem status +const bool useCTSforStatus = false; // Flag to use the modem CTS pin for status +const int8_t modemResetPin = 20; // MCU pin connected to modem reset pin +const int8_t modemSleepRqPin = 23; // MCU pin for modem sleep/wake request +const int8_t modemLEDPin = redLED; // MCU pin connected an LED to show modem + // status (-1 if unconnected) + +// Network connection information +const char* apn = + "hologram"; // APN connection name, typically Hologram unless you have a + // different provider's SIM card. Change as needed + +DigiXBeeCellularTransparent modemXBCT(&modemSerial, modemVccPin, modemStatusPin, + useCTSforStatus, modemResetPin, + modemSleepRqPin, apn); +// Create an extra reference to the modem by a generic name +DigiXBeeCellularTransparent modem = modemXBCT; +/** End [digi_xbee_cellular_transparent] */ + + +// ========================================================================== +// Using the Processor as a Sensor +// ========================================================================== +/** Start [processor_sensor] */ +#include + +// Create the main processor chip "sensor" - for general metadata +const char* mcuBoardVersion = "v0.5b"; +ProcessorStats mcuBoard(mcuBoardVersion); +/** End [processor_sensor] */ + + +// ========================================================================== +// Maxim DS3231 RTC (Real Time Clock) +// ========================================================================== +/** Start [ds3231] */ +#include + +// Create a DS3231 sensor object +MaximDS3231 ds3231(1); +/** End [ds3231] */ + + + +// ========================================================================== +// Creating the Variable Array[s] and Filling with Variable Objects +// ========================================================================== +/** Start [variable_arrays] */ +Variable* variableList[] = { + // new ProcessorStats_Battery(&mcuBoard), + new MaximDS3231_Temp(&ds3231), + // new Modem_SignalPercent(&modem), +}; + +// All UUID's, device registration, and sampling feature information can be +// pasted directly from Monitor My Watershed. +// To get the list, click the "View token UUID list" button on the upper right +// of the site page. + +// *** CAUTION --- CAUTION --- CAUTION --- CAUTION --- CAUTION *** +// Check the order of your variables in the variable list!!! +// Be VERY certain that they match the order of your UUID's! +// Rearrange the variables in the variable list ABOVE if necessary to match! +// Do not change the order of the variables in the section below. +// *** CAUTION --- CAUTION --- CAUTION --- CAUTION --- CAUTION *** + +// Replace all of the text in the following section with the UUID array from +// MonitorMyWatershed + +/* clang-format off */ +// --------------------- Beginning of Token UUID List --------------------- + + +const char* UUIDs[] = // UUID array for device sensors + { + // formatted below to all have 11 characters + // "Mayfly_Batt", // Battery voltage (EnviroDIY_Mayfly_Batt) + "measurement", // Board Temperature (EnviroDIY_Mayfly_Temp) + // "LTEB_Signal", // Percent full scale (EnviroDIY_LTEB_SignalPercent) +}; +const char* registrationToken = "SharedAccessSignature sr=https%3A%2F%2Fevent-hub-data-logger.servicebus.windows.net%2Fdevices%2Fmessages&sig=c9JbL/90pYNuGVOPx7pcsk2xtYvlcUSVPS5td8Uqgk0%3D&se=1650395209&skn=mayfly-device"; // Device registration token +const char* samplingFeature = "27abab02-2c22-452e-8c26-3bce138554ee"; // Sampling feature UUID + + +// ----------------------- End of Token UUID List ----------------------- +/* clang-format on */ + +// Count up the number of pointers in the array +int variableCount = sizeof(variableList) / sizeof(variableList[0]); + +// Create the VariableArray object +VariableArray varArray(variableCount, variableList, UUIDs); +/** End [variable_arrays] */ + + +// ========================================================================== +// The Logger Object[s] +// ========================================================================== +/** Start [loggers] */ +// Create a new logger instance +Logger dataLogger(LoggerID, loggingInterval, &varArray); +/** End [loggers] */ + + +// ========================================================================== +// Creating Data Publisher[s] +// ========================================================================== +/** Start [publishers] */ +// Create a data publisher for the Monitor My Watershed/EnviroDIY POST endpoint +#include +EventHubPublisher EnviroDIYPOST(dataLogger, &modem.gsmClient, + registrationToken, samplingFeature); +/** End [publishers] */ + + +// ========================================================================== +// Working Functions +// ========================================================================== +/** Start [working_functions] */ +// Flashes the LED's on the primary board +void greenredflash(uint8_t numFlash = 4, uint8_t rate = 75) { + for (uint8_t i = 0; i < numFlash; i++) { + digitalWrite(greenLED, HIGH); + digitalWrite(redLED, LOW); + delay(rate); + digitalWrite(greenLED, LOW); + digitalWrite(redLED, HIGH); + delay(rate); + } + digitalWrite(redLED, LOW); +} + +// Reads the battery voltage +// NOTE: This will actually return the battery level from the previous update! +float getBatteryVoltage() { + if (mcuBoard.sensorValues[0] == -9999) mcuBoard.update(); + return mcuBoard.sensorValues[0]; +} + + +// ========================================================================== +// Arduino Setup Function +// ========================================================================== +/** Start [setup] */ +void setup() { + // Start the primary serial connection + Serial.begin(serialBaud); + + // Print a start-up note to the first serial port + Serial.print(F("Now running ")); + Serial.print(sketchName); + Serial.print(F(" on Logger ")); + Serial.println(LoggerID); + Serial.println(); + + Serial.print(F("Using ModularSensors Library version ")); + Serial.println(MODULAR_SENSORS_VERSION); + Serial.print(F("TinyGSM Library version ")); + Serial.println(TINYGSM_VERSION); + Serial.println(); + + // Start the serial connection with the modem + modemSerial.begin(modemBaud); + + // Set up pins for the LED's + pinMode(greenLED, OUTPUT); + digitalWrite(greenLED, LOW); + pinMode(redLED, OUTPUT); + digitalWrite(redLED, LOW); + // Blink the LEDs to show the board is on and starting up + greenredflash(); + + pinMode(20, OUTPUT); // for proper operation of the onboard flash memory + // chip's ChipSelect (Mayfly v1.0 and later) + + // Set the timezones for the logger/data and the RTC + // Logging in the given time zone + Logger::setLoggerTimeZone(timeZone); + // It is STRONGLY RECOMMENDED that you set the RTC to be in UTC (UTC+0) + Logger::setRTCTimeZone(0); + + // Attach the modem and information pins to the logger + dataLogger.attachModem(modem); + modem.setModemLED(modemLEDPin); + dataLogger.setLoggerPins(wakePin, sdCardSSPin, sdCardPwrPin, buttonPin, + greenLED); + + // Begin the logger + dataLogger.begin(); + + // Test HMAC token + // Secret key and Plain Text to Compute Hash + const char* key = "Jefe"; + const char* text_to_hash = "what do ya want for nothing?"; + + // Call method of dataPublisher object + EnviroDIYPOST.writeHMACtoken(key, text_to_hash); + + + // Note: Please change these battery voltages to match your battery + // Set up the sensors, except at lowest battery level + if (getBatteryVoltage() > 3.4) { + Serial.println(F("Setting up sensors...")); + varArray.setupSensors(); + } + + // Extra modem set-up - selecting AT&T as the carrier and LTE-M only + // NOTE: The code for this could be shortened using the "commandMode" and + // other XBee specific commands in TinyGSM. I've written it this way in + // this example to show how the settings could be changed in either bypass + // OR transparent mode. + Serial.println(F("Waking modem and setting Cellular Carrier Options...")); + modem.modemWake(); // NOTE: This will also set up the modem + // Go back to command mode to set carrier options + for (uint8_t i = 0; i < 5; i++) { + // Wait the required guard time before entering command mode + delay(1010); + modem.gsmModem.streamWrite(GF("+++")); // enter command mode + if (modem.gsmModem.waitResponse(2000, GF("OK\r")) == 1) break; + } + // Carrier Profile - 0 = Automatic selection + // - 1 = No profile/SIM ICCID selected + // - 2 = AT&T + // - 3 = Verizon + // NOTE: To select T-Mobile, you must enter bypass mode! + modem.gsmModem.sendAT(GF("CP"), 2); + modem.gsmModem.waitResponse(GF("OK\r")); + // Cellular network technology - 0 = LTE-M with NB-IoT fallback + // - 1 = NB-IoT with LTE-M fallback + // - 2 = LTE-M only + // - 3 = NB-IoT only + modem.gsmModem.sendAT(GF("N#"), 2); + modem.gsmModem.waitResponse(); + // Write changes to flash and apply them + Serial.println(F("Wait while applying changes...")); + // Write changes to flash + modem.gsmModem.sendAT(GF("WR")); + modem.gsmModem.waitResponse(GF("OK\r")); + // Apply changes + modem.gsmModem.sendAT(GF("AC")); + modem.gsmModem.waitResponse(GF("OK\r")); + // Reset the cellular component to ensure network settings are changed + modem.gsmModem.sendAT(GF("!R")); + modem.gsmModem.waitResponse(30000L, GF("OK\r")); + // Force reset of the Digi component as well + // This effectively exits command mode + modem.gsmModem.sendAT(GF("FR")); + modem.gsmModem.waitResponse(5000L, GF("OK\r")); + + // Sync the clock if it isn't valid or we have battery to spare + if (getBatteryVoltage() > 3.55 || !dataLogger.isRTCSane()) { + // Synchronize the RTC with NIST + // This will also set up the modem + dataLogger.syncRTC(); + } + + // Create the log file, adding the default header to it + // Do this last so we have the best chance of getting the time correct and + // all sensor names correct + // Writing to the SD card can be power intensive, so if we're skipping + // the sensor setup we'll skip this too. + if (getBatteryVoltage() > 3.4) { + Serial.println(F("Setting up file on SD card")); + dataLogger.turnOnSDcard( + true); // true = wait for card to settle after power up + dataLogger.createLogFile(true); // true = write a new header + dataLogger.turnOffSDcard( + true); // true = wait for internal housekeeping after write + } + + // Call the processor sleep + Serial.println(F("Putting processor to sleep\n")); + dataLogger.systemSleep(); +} +/** End [setup] */ + + +// ========================================================================== +// Arduino Loop Function +// ========================================================================== +/** Start [loop] */ +// Use this short loop for simple data logging and sending +void loop() { + // Note: Please change these battery voltages to match your battery + // At very low battery, just go back to sleep + if (getBatteryVoltage() < 3.4) { + dataLogger.systemSleep(); + } + // At moderate voltage, log data but don't send it over the modem + else if (getBatteryVoltage() < 3.55) { + dataLogger.logData(); + } + // If the battery is good, send the data to the world + else { + dataLogger.logDataAndPublish(); + } +} +/** End [loop] */ diff --git a/sensor_tests/HMACEventHubTest_Digi/platformio.ini b/sensor_tests/HMACEventHubTest_Digi/platformio.ini new file mode 100644 index 000000000..37f1d11cf --- /dev/null +++ b/sensor_tests/HMACEventHubTest_Digi/platformio.ini @@ -0,0 +1,41 @@ +; PlatformIO Project Configuration File +; +; Build options: build flags, source filter +; Upload options: custom upload port, speed and extra flags +; Library options: dependencies, extra library storages +; Advanced options: extra scripting +; +; Please visit documentation for the other options and examples +; http://docs.platformio.org/page/projectconf.html + +[platformio] +description = Test HMAC & POST Requests to Event Hub API, using Digi LTE-M radio & Mayfly 0.5b +src_dir = sensor_tests/HMACEventHubTest_Digi + +[env:mayfly] +monitor_speed = 115200 +board = mayfly +platform = atmelavr +framework = arduino +lib_ldf_mode = deep+ +lib_ignore = + RTCZero + Adafruit NeoPixel + Adafruit GFX Library + Adafruit SSD1306 + Adafruit ADXL343 + Adafruit STMPE610 + Adafruit TouchScreen + Adafruit ILI9341 +build_flags = + -D SDI12_EXTERNAL_PCINT + -D TINY_GSM_RX_BUFFER=512 + -D TINY_GSM_YIELD_MS=2 + -D MS_DATAPUBLISHERBASE_DEBUG + -D MS_EVENTHUBPUBLISHER_DEBUG + ; -D TINY_GSM_DEBUG=Serial + ; -D MS_LOGGERMODEM_DEBUG + ; -D MS_DIGIXBEECELLULARTRANSPARENT_DEBUG +lib_deps = + https://github.com/EnviroDIY/ModularSensors.git#hmac_auth +; ^^ Use this when if you want to pull from the hmac_auth feature branch diff --git a/src/publishers/EventHubPublisher.cpp b/src/publishers/EventHubPublisher.cpp index 4b584926f..e4219eb22 100644 --- a/src/publishers/EventHubPublisher.cpp +++ b/src/publishers/EventHubPublisher.cpp @@ -18,7 +18,7 @@ // Constant values for post requests // I want to refer to these more than once while ensuring there is only one copy // in memory -const char* EventHubPublisher::postEndpoint = "https://event-hub-data-logger.servicebus.windows.net/devices/messages?timeout=60"; +const char* EventHubPublisher::postEndpoint = "https://event-hub-data-logger.servicebus.windows.net/devices/messages"; const char* EventHubPublisher::eventHubHost = "event-hub-data-logger.servicebus.windows.net"; const int EventHubPublisher::eventHubPort = 443; // 443 for HTTPS; 80 for HTTP const char* EventHubPublisher::tokenHeader = "\r\nAuthorization: "; @@ -265,6 +265,7 @@ int16_t EventHubPublisher::publishData(Client* outClient) { txBuffer[strlen(txBuffer)] = '}'; } } + Serial.print(F("\n")); // Send out the finished request (or the last unsent section of it) printTxBuffer(outClient, true); From 8bc473c7b423dcb56a9c7582300efb511b435479 Mon Sep 17 00:00:00 2001 From: Anthony Aufdenkampe Date: Tue, 3 May 2022 14:39:28 -0500 Subject: [PATCH 7/8] EventHub working with `TinyGsmClientSecure` Commiting work from 4/20/22. The key was to use `TinyGsmClientSecure` rather than `TinyGsmClient` in src/modems/SIMComSIM7080.h --- sensor_tests/HMACEventHubTest/HMACEventHubTest.ino | 6 +++--- .../HMACEventHubTest_Digi/HMACEventHubTest_Digi.ino | 4 ++-- src/modems/SIMComSIM7080.h | 11 +++++++++-- src/publishers/EventHubPublisher.cpp | 3 +++ src/publishers/EventHubPublisher.h | 3 +++ 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino b/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino index f8b3b7eff..5acd5c087 100644 --- a/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino +++ b/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino @@ -175,7 +175,7 @@ const char* UUIDs[] = // UUID array for device sensors "measurement", // Board Temperature (EnviroDIY_Mayfly_Temp) // "LTEB_Signal", // Percent full scale (EnviroDIY_LTEB_SignalPercent) }; -const char* registrationToken = "SharedAccessSignature sr=https%3A%2F%2Fevent-hub-data-logger.servicebus.windows.net%2Fdevices%2Fmessages&sig=pDocq7bRcZpKgt%2BWmNQHsJjz36mcssxF0EQ6jnVjd/g%3D&se=1650317059&skn=mayfly-device"; // Device registration token +const char* registrationToken = "SharedAccessSignature sr=https%3A%2F%2Fevent-hub-data-logger.servicebus.windows.net%2Fdevices%2Fmessages&sig=KCT4Hdnh2iL2prxOsKO03RFHqhho5VTokIZHrGPYpaQ%3D&se=1650474173&skn=mayfly-device"; // Device registration token const char* samplingFeature = "7d37e135-0e26-4bc7-aa81-f9443283582d"; // Sampling feature UUID @@ -205,7 +205,7 @@ Logger dataLogger(LoggerID, loggingInterval, &varArray); /** Start [publishers] */ // Create a data publisher for the Monitor My Watershed/EnviroDIY POST endpoint #include -EventHubPublisher EnviroDIYPOST(dataLogger, &modem.gsmClient, +EventHubPublisher EventHubPOST(dataLogger, &modem.gsmClient, registrationToken, samplingFeature); /** End [publishers] */ @@ -291,7 +291,7 @@ void setup() { const char* text_to_hash = "what do ya want for nothing?"; // Call method of dataPublisher object - EnviroDIYPOST.writeHMACtoken(key, text_to_hash); + EventHubPOST.writeHMACtoken(key, text_to_hash); // Note: Please change these battery voltages to match your battery diff --git a/sensor_tests/HMACEventHubTest_Digi/HMACEventHubTest_Digi.ino b/sensor_tests/HMACEventHubTest_Digi/HMACEventHubTest_Digi.ino index d98bb00d5..1edf13722 100644 --- a/sensor_tests/HMACEventHubTest_Digi/HMACEventHubTest_Digi.ino +++ b/sensor_tests/HMACEventHubTest_Digi/HMACEventHubTest_Digi.ino @@ -206,7 +206,7 @@ Logger dataLogger(LoggerID, loggingInterval, &varArray); /** Start [publishers] */ // Create a data publisher for the Monitor My Watershed/EnviroDIY POST endpoint #include -EventHubPublisher EnviroDIYPOST(dataLogger, &modem.gsmClient, +EventHubPublisher EventHubPOST(dataLogger, &modem.gsmClient, registrationToken, samplingFeature); /** End [publishers] */ @@ -292,7 +292,7 @@ void setup() { const char* text_to_hash = "what do ya want for nothing?"; // Call method of dataPublisher object - EnviroDIYPOST.writeHMACtoken(key, text_to_hash); + EventHubPOST.writeHMACtoken(key, text_to_hash); // Note: Please change these battery voltages to match your battery diff --git a/src/modems/SIMComSIM7080.h b/src/modems/SIMComSIM7080.h index 26a1d643d..ffedacfd3 100644 --- a/src/modems/SIMComSIM7080.h +++ b/src/modems/SIMComSIM7080.h @@ -199,9 +199,16 @@ class SIMComSIM7080 : public loggerModem { */ TinyGsm gsmModem; /** - * @brief Public reference to the TinyGSM Client. + * @brief Public reference to the TinyGSM Client (HTTP). */ - TinyGsmClient gsmClient; + TinyGsmClientSecure gsmClient; + // /** + // * @brief Public reference to a secure TinyGSM Client (HTTPS). + // */ + // #ifdef MS_HTTPS + // TinyGsmClientSecure gsmClient; + // #endif + protected: bool isInternetAvailable(void) override; diff --git a/src/publishers/EventHubPublisher.cpp b/src/publishers/EventHubPublisher.cpp index e4219eb22..e7a3d243b 100644 --- a/src/publishers/EventHubPublisher.cpp +++ b/src/publishers/EventHubPublisher.cpp @@ -199,6 +199,9 @@ int16_t EventHubPublisher::publishData(Client* outClient) { MS_START_DEBUG_TIMER; if (outClient->connect(eventHubHost, eventHubPort)) { MS_DBG(F("Client connected after"), MS_PRINT_DEBUG_TIMER, F("ms\n")); + #ifdef MS_HTTPS + MS_DBG(F("Performing HTTPS POST request... ")); + #endif // copy the initial post header into the tx buffer strcpy(txBuffer, postHeader); diff --git a/src/publishers/EventHubPublisher.h b/src/publishers/EventHubPublisher.h index 89173407c..942034375 100644 --- a/src/publishers/EventHubPublisher.h +++ b/src/publishers/EventHubPublisher.h @@ -21,6 +21,9 @@ #define MS_DEBUGGING_STD "EventHubPublisher" #endif +// Specify HTTPS Secure Client for this Publisher +#define MS_HTTPS + // Included Dependencies #include "ModSensorDebugger.h" #undef MS_DEBUGGING_STD From 0f2e07cb489eae520f69d1e9adc6099069f28167 Mon Sep 17 00:00:00 2001 From: Anthony Aufdenkampe Date: Sat, 11 Jun 2022 11:31:44 -0500 Subject: [PATCH 8/8] Getting HMAC into dataPublisher base --- .gitignore | 1 + sensor_tests/HMACEventHubTest/HMACEventHubTest.ino | 9 ++++++++- src/dataPublisherBase.cpp | 14 +++----------- src/dataPublisherBase.h | 6 +++--- src/publishers/EventHubPublisher.h | 2 ++ 5 files changed, 17 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index bf63dcacd..d487d35a6 100644 --- a/.gitignore +++ b/.gitignore @@ -94,3 +94,4 @@ cache docs/output_copyFunctions.log docs/output_documentExamples.log docs/output_fixSectionsInXml.log +*HMAC_secret_key.h diff --git a/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino b/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino index 5acd5c087..cf56cdf70 100644 --- a/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino +++ b/sensor_tests/HMACEventHubTest/HMACEventHubTest.ino @@ -12,7 +12,7 @@ * and the EnviroDIY Development Team * This example is published under the BSD-3 license. * - + * Software: ModularSensors, `hmac_auth` branch (based on v0.33.3 or later) * Hardware Platform: EnviroDIY Mayfly Arduino Datalogger * * DISCLAIMER: @@ -284,6 +284,13 @@ void setup() { // Begin the logger dataLogger.begin(); + + // Test SAS token (signature) generation + // Print shared secret key + Serial.println(F("Testing SAS token/signature generation")); + Serial.print(F("HMAC Secret Key: ")); + Serial.println(SECRET_KEY); + Serial.println(); // Test HMAC token // Secret key and Plain Text to Compute Hash diff --git a/src/dataPublisherBase.cpp b/src/dataPublisherBase.cpp index a1515d9d4..ccf2ab5c9 100644 --- a/src/dataPublisherBase.cpp +++ b/src/dataPublisherBase.cpp @@ -157,22 +157,14 @@ String dataPublisher::parseMQTTState(int state) { } -String dataPublisher::writeHMACtoken(const char* key, const char* string_to_sign) { +uint8_t* dataPublisher::writeHMACtoken(const char* key, const char* string_to_sign) { MS_DBG(F("Writing an HMAC-SHA256 token")); - // Anthony to add more code here // Initialize the Sha256Wrapper for HMAC hashing, "salting" with the secret key // Must be invoked before hashing. Sha256.initHmac((uint8_t*)key, strlen(key)); // Recasting char* key to uint8_t* // Write data into the hasher. Sha256.write(string_to_sign); // equivalent to `Sha256.print()` // Returns a reference to the hash. Once this method has been called init must be invoked again. - uint8_t* result = Sha256.resultHmac(); - // Print 32-byte hash as hexidecimal string (64 characters) - Serial.print(F("Token as Hex: ")); - for (int i = 0; i < 32; i++) { - if (result[i] < 16) Serial.print("0"); // Otherwise leading zeros will be dropped - Serial.print(result[i],HEX); - } - Serial.print(F("\n\n")); - return "token"; // eventually return token + uint8_t* token = Sha256.resultHmac(); + return token; // return 32-byte token } diff --git a/src/dataPublisherBase.h b/src/dataPublisherBase.h index 24b9679ba..6df257cba 100644 --- a/src/dataPublisherBase.h +++ b/src/dataPublisherBase.h @@ -277,10 +277,10 @@ class dataPublisher { * * @param key The shared secret key used to "salt" the hash * @param string_to_sign The string that gets hashed into a signature token. - * @return **String** The signed HMAC-SHA256 authorization token, or - * signature. + * @return **uint8_t*** The signed 32-byte HMAC-SHA256 authorization token, + * or signature. */ - String writeHMACtoken(const char* key, const char* string_to_sign); + uint8_t* writeHMACtoken(const char* key, const char* string_to_sign); protected: diff --git a/src/publishers/EventHubPublisher.h b/src/publishers/EventHubPublisher.h index 942034375..85e6e3254 100644 --- a/src/publishers/EventHubPublisher.h +++ b/src/publishers/EventHubPublisher.h @@ -28,6 +28,8 @@ #include "ModSensorDebugger.h" #undef MS_DEBUGGING_STD #include "dataPublisherBase.h" +// Pull in Secret Key value from a file that is not tracked by Git +#include "/Users/aaufdenkampe/Documents/Arduino/EnviroDIY_ModularSensors/sensor_tests/HMACEventHubTest/HMAC_secret_key.h" // ============================================================================