Skip to content

Commit

Permalink
Added Publisher Pacing - #21
Browse files Browse the repository at this point in the history
  • Loading branch information
neilh20.gitk authored and neilh20.gitk committed Dec 14, 2020
1 parent b25cd2c commit 936ecef
Show file tree
Hide file tree
Showing 10 changed files with 262 additions and 49 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ Temporary Items

# Atom / PlatformIO
.pio
.pio_del
.pioenvs
.piolibdeps
.pio
Expand Down
52 changes: 27 additions & 25 deletions a/tu_xx01/platformio.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,19 @@ default_envs = mayfly
; Ignore these folders or PlatformIO will double count all the dependencies
; Development: ModularSensors Release1: src ?
; ?? .pioenvs, .piolibdeps, .vscode, include, doc, examples, sensor_tests, compile_tests, pioScripts
lib_ignore = .git, doc, examples, arduino_update, ModularSensors
lib_ignore = .git, doc, examples, arduino_update, ModularSensors, sensor_tests

lib_cmn_deps =
; These are latest beyond EnviroDIY releases
https://github.com/vshymanskyy/StreamDebugger ;Debug when needed
; StreamDebugger ; Same as above
; del https://github.com/neilh10/SensorModbusMaster ; default release1
https://github.com/enviroDIY/SensorModbusMaster
;https://github.com/enviroDIY/SensorModbusMaster
https://github.com/neilh10/Adafruit_INA219.git
; del https://github.com/adafruit/SdFat ;- need adafruit/SdFat for multiple SDx
https://github.com/greiman/SdFat.git ;-
;https://github.com/greiman/SdFat.git ;-
;https://github.com/neilh10/EnableInterrupt - old v1.0.0 need v1.1
https://github.com/GreyGnome/EnableInterrupt
;https://github.com/GreyGnome/EnableInterrupt
; del https://github.com/arduino-libraries/NTPClient - repalced
;; see lib_samd_deps for Adafruit_SPIFlash & Adafruit_TinyUSB_Arduino & Adafruit-GFX-Library Adafruit_NeoPixel
;
Expand All @@ -41,25 +41,25 @@ lib_cmn_deps =
;https://github.com/neilh10/ModularSensors#rel1_dvlp1m
; ^^ Use this when you want to pull from a specific branch
; STD_LIBS pulled in by ModularSensors, needed if not enabling ModularSensors to Historical Ref
https://github.com/soligen2010/Adafruit_ADS1X15
https://github.com/adafruit/Adafruit_AM2315
https://github.com/adafruit/Adafruit_LC709203F
;https://github.com/soligen2010/Adafruit_ADS1X15
;https://github.com/adafruit/Adafruit_AM2315
;https://github.com/adafruit/Adafruit_LC709203F
;https://github.com/adafruit/Adafruit_AM2320
https://github.com/adafruit/Adafruit_BME280_Library
https://github.com/adafruit/Adafruit_MPL115A2
https://github.com/adafruit/DHT-sensor-library
https://github.com/adafruit/Adafruit_Sensor
https://github.com/milesburton/Arduino-Temperature-Control-Library ;Pulls in DallasTemperature
;https://github.com/adafruit/Adafruit_BME280_Library
;https://github.com/adafruit/Adafruit_MPL115A2
;lhttps://github.com/adafruit/DHT-sensor-library
;https://github.com/adafruit/Adafruit_Sensor
;https://github.com/milesburton/Arduino-Temperature-Control-Library ;Pulls in DallasTemperature
;https://github.com/neilh10/KellerModbus
https://github.com/EnviroDIY/KellerModbus
https://github.com/NorthernWidget/MS5803
;https://github.com/EnviroDIY/KellerModbus
;https://github.com/NorthernWidget/MS5803
;https://github.com/PaulStoffregen/OneWire
https://github.com/knolleary/pubsubclient.git
;https://github.com/knolleary/pubsubclient.git
;bug https://github.com/EnviroDIY/TinyGSM.git ; bug EnviroDIY/ModularSensors/issues 311
https://github.com/neilh10/TinyGSM.git ; Envirodiy goodenough
https://github.com/EnviroDIY/YosemitechModbus
https://github.com/EnviroDIY/Arduino-SDI-12.git ;https://github.com/neilh10/Arduino-SDI-12 ;release1 set as default
https://github.com/EnviroDIY/Tally_Library.git#Dev_I2C
;https://github.com/EnviroDIY/YosemitechModbus
;https://github.com/EnviroDIY/Arduino-SDI-12.git ;https://github.com/neilh10/Arduino-SDI-12 ;release1 set as default
;https://github.com/EnviroDIY/Tally_Library.git#Dev_I2C
;
;Historical reference
;https://github.com/greiman/SdFat.git ;- need adafruit/SdFat for multiple SDx
Expand Down Expand Up @@ -133,16 +133,18 @@ build_flags =
-fmax-errors=5

lib_deps =${common.lib_cmn_deps} ${common.lib_avr_deps}
;EnviroDIY_ModularSensors
; ^^ Use this when working from an official release of the library
; https://github.com/EnviroDIY/ModularSensors.git#develop
; ^^ Use this when if you want to pull from the develop branch
; for clean lib - delete .pio, uncomment below
; pio lib install - execute
; then recomment below
;https://github.com/EnviroDIY/ModularSensors.git#develop
; This uses library.json
; Then to not have conflicts with ../../src delete .pio/libdeps/xxx/EnviroDIY_modularSensors h
https://github.com/neilh10/AltSoftSerial ; Use this for managing pwr - data pin off as well.
;https://github.com/PaulStoffregen/AltSoftSerial.git
;
https://github.com/EnviroDIY/SoftwaterSerial_ExternalInts.git
https://github.com/SRGDamia1/NeoSWSerial.git
https://github.com/EnviroDIY/Sodaq_DS3231.git
;https://github.com/EnviroDIY/SoftwaterSerial_ExternalInts.git
;https://github.com/SRGDamia1/NeoSWSerial.git
;https://github.com/EnviroDIY/Sodaq_DS3231.git

monitor_filters = log2file, default, time
;
Expand Down
158 changes: 158 additions & 0 deletions a/tu_xx01/platformio.ini_updated
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
; PlatformIO Project Configuration File
;
; Build options: build flags, source filter
; Upload options: custom upload port, speed and extra flags
; Library options: dependencies, extra library storages
; Advanced options: extra scripting
;
; Please visit documentation for the other options and examples
; http://docs.platformio.org/page/projectconf.html

[platformio]
description = neilh10/ModularSensors tu_ctd logging to Monitor My Watershed
;src_dir =.
default_envs = mayfly

[common]
; Ignore these folders or PlatformIO will double count all the dependencies
; Development: ModularSensors Release1: src ?
; ?? .pioenvs, .piolibdeps, .vscode, include, doc, examples, sensor_tests, compile_tests, pioScripts
lib_ignore = .git, doc, examples, arduino_update, ModularSensors, sensor_tests

lib_cmn_deps =
; These are latest beyond EnviroDIY releases
https://github.com/vshymanskyy/StreamDebugger ;Debug when needed
; StreamDebugger ; Same as above
; del https://github.com/neilh10/SensorModbusMaster ; default release1
https://github.com/enviroDIY/SensorModbusMaster
https://github.com/neilh10/Adafruit_INA219.git
; del https://github.com/adafruit/SdFat ;- need adafruit/SdFat for multiple SDx
https://github.com/greiman/SdFat.git ;-
;https://github.com/neilh10/EnableInterrupt - old v1.0.0 need v1.1
;https://github.com/GreyGnome/EnableInterrupt
; del https://github.com/arduino-libraries/NTPClient - repalced
;; see lib_samd_deps for Adafruit_SPIFlash & Adafruit_TinyUSB_Arduino & Adafruit-GFX-Library Adafruit_NeoPixel
;
; For development, disable ModularSensors and enable cmn_src_filter
; Need extra libs, so eable STD_LIBS or 1st pass enable ModularSensors then build pulling in ref libs,
;EnviroDIY_ModularSensors
; ^^ Use this when working from an official release of the library
;https://github.com/neilh10/ModularSensors#release1
;https://github.com/neilh10/ModularSensors#rel1_dvlp1m
; ^^ Use this when you want to pull from a specific branch
; STD_LIBS pulled in by ModularSensors, needed if not enabling ModularSensors to Historical Ref
;https://github.com/soligen2010/Adafruit_ADS1X15
;https://github.com/adafruit/Adafruit_AM2315
https://github.com/adafruit/Adafruit_LC709203F
;https://github.com/adafruit/Adafruit_AM2320
;https://github.com/adafruit/Adafruit_BME280_Library
;https://github.com/adafruit/Adafruit_MPL115A2
;https://github.com/adafruit/DHT-sensor-library
;https://github.com/adafruit/Adafruit_Sensor
;https://github.com/milesburton/Arduino-Temperature-Control-Library ;Pulls in DallasTemperature
;https://github.com/neilh10/KellerModbus ;check for mods
;https://github.com/EnviroDIY/KellerModbus
;https://github.com/NorthernWidget/MS5803
;https://github.com/PaulStoffregen/OneWire
;https://github.com/knolleary/pubsubclient.git
;bug https://github.com/EnviroDIY/TinyGSM.git ; bug EnviroDIY/ModularSensors/issues 311
https://github.com/neilh10/TinyGSM.git ; Envirodiy goodenough
;https://github.com/EnviroDIY/YosemitechModbus
;https://github.com/EnviroDIY/Arduino-SDI-12
;https://github.com/neilh10/Arduino-SDI-12 ;release1 set as default
;https://github.com/EnviroDIY/Tally_Library.git#Dev_I2C
;
;Historical reference
;https://github.com/greiman/SdFat.git ;- need adafruit/SdFat for multiple SDx

lib_avr_deps = ;Specific Mayfly board goes in [Mayfly]

; for development, can edit directly in these directories ... relative to src_dir
; Use the src filter to ensure subfolders are built
; If enabled, disable in lib_cmn_deps=[]ModularSensors, and may need to delete duplicate directory .libdeps/ModularSensors
; Also srcFilter in libray.json?
cmn_src_filter = +<*> +<../../../src> +<../../../src/sensors> +<../../../src/publishers> +<../../../src/modems>

#build_flags -v for detailed cc verbose
cmn_build_flags =
-Isrc
-I../../src ; .h For easy debug
-I../../src/sensors ; .h For easy debug
!python git_rev_macro.py ;returns -DPIO_SRC_REV=<git_revision>
;-DTINY_GSM_DEBUG=Serial
;-DMQTT_MAX_PACKET_SIZE=240
;-DMS_TU_CTD_DEBUG
;-DMS_TU_CTD_DEBUG_DEEP
;-DNO_FIRST_SYNC_WITH_NIST
;-DSTREAMDEBUGGER_DBG
-DMS_LOGGERBASE_POSTS
;-DMS_LOGGERBASE_SLEEP_DEBUG ;Need or below
;-DMS_LOGGERBASE_DEBUG ; ALso debugging output when sleeping
;-DMS_LOGGERBASE_DEEP_DEBUG
;-DMS_PROCESSORSTATS_DEBUG
;-DMS_PROCESSORADC_DEBUG
;-DMS_LOGGERMODEM_DEBUG
;-DMS_LOGGERMODEM_DEBUG_DEEP
;-DMS_DATAPUBLISHERBASE_DEBUG
;-DMS_ENVIRODIYPUBLISHER_DEBUG
;-DMS_DIGIXBEEWIFI_DEBUG
;-DMS_DIGIXBEEWIFI_DEBUG_DEEP
;-DMS_DIGIXBEECELLULARTRANSPARENT_DEBUG
;-DMS_DIGIXBEECELLULARTRANSPARENT_DEBUG_DEEP
;-DMS_DIGIXBEE_DEBUG
;-DMS_SENSORBASE_DEBUG
;-DMS_TIINA219M_DEBUG
;-DMS_AOSONGAM2315_DEBUG
;-DMS_EXTERNALVOLTAGE_DEBUG
;-DMS_EXTERNALVOLTAGE_DEBUG_DEEP
;-DMS_VARIABLEARRAY_DEBUG
;-DMS_VARIABLEARRAY_DEBUG_DEEP
;-DMS_VARIABLEBASE_DEBUG
;-DMS_VARIABLEBASE_DEBUG_DEEP
;-DMS_KELLERPARENT_DEBUG
;-DMS_KELLERPARENT_DEBUG_DEEP has problem
;-DMS_MODBUS_DEBUG
;-DMS_ANALOGELECCONDUCTIVITY_DEBUG
;-DMS_ANALOGELECCONDUCTIVITY_DEBUG_DEEP
;-DMS_SDI12SENSORS_DEBUG
;-DMS_ENVIRODIYPUBLISHER_DEBUG
;-DMS_ENVIRODIYPUBLISHER_DEBUG_DEEP

[env:mayfly]
;upload_port = COM20
monitor_speed = 115200
board = mayfly
platform = atmelavr
framework = arduino
lib_compat_mode = strict ; Debugging multiple TinyGSMs
lib_ldf_mode = deep+
lib_ignore = ${common.lib_ignore}, RTCZero
src_filter = ${common.cmn_src_filter}
build_flags =
${common.cmn_build_flags}
-DSDI12_EXTERNAL_PCINT
-DNEOSWSERIAL_EXTERNAL_PCINT
-fmax-errors=5

lib_deps =
${common.lib_cmn_deps} ${common.lib_avr_deps}
;EnviroDIY_ModularSensors
; ^^ Use this when working from an official release of the library
;https://github.com/EnviroDIY/ModularSensors.git#develop
; ^^ Use this when if you want to pull from the develop branch
https://github.com/neilh10/AltSoftSerial ; Use this for managing pwr - data pin off as well.
;https://github.com/PaulStoffregen/AltSoftSerial.git
;
;https://github.com/EnviroDIY/SoftwaterSerial_ExternalInts.git
;https://github.com/SRGDamia1/NeoSWSerial.git
;https://github.com/EnviroDIY/Sodaq_DS3231.git

monitor_filters = log2file, default, time
;
; The following monitor_flags setting is needed becasue in transparent mode the XBee3
; uses CR line endings instead of the more typical CR+LF. This setting enables you
; to see all back and forth communication.
; (https://www.envirodiy.org/topic/connecting-xbee3-lte-to-the-internet/#post-13312)
monitor_flags =
--eol
CR
4 changes: 2 additions & 2 deletions a/tu_xx01/src/tu_xx01.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1113,8 +1113,8 @@ void setup() {
#define POLL_MODEM_REQ \
(loggerModem::PollModemMetaData_t)( \
loggerModem::POLL_MODEM_META_DATA_RSSI | \
loggerModem::POLL_MODEM_META_DATA_VCC)
modemPhy.pollModemMetadata(POLL_MODEM_REQ);
loggerModem::POLL_MODEM_META_DATA_VCC )
modemPhy.pollModemMetadata(loggerModem::POLL_MODEM_META_DATA_RSSI );
#endif
#endif // UseModem_Module
dataLogger.setLoggerPins(wakePin, sdCardSSPin, sdCardPwrPin, buttonPin,
Expand Down
21 changes: 20 additions & 1 deletion src/LoggerBaseExtCpp.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -826,6 +826,8 @@ void Logger::publishDataQuedToRemotes(bool internetPresent) {
*/

if (dataPublishers[i]->getQuedStatus()) {
uint16_t delay_posted_pacing_ms = dataPublishers[i]->getTimerPostPacing_mS();
uint16_t published_this_pass =0;
serzQuedStart((char)('0' + i));
deszRdelStart();
// MS_START_DEBUG_TIMER;
Expand Down Expand Up @@ -870,6 +872,14 @@ void Logger::publishDataQuedToRemotes(bool internetPresent) {
}
*/
} else {
/*A publish has been sucessfull.
* Slow Down sending based on publishers acceptance rate
* Each publish creates and tears down a TCP connection */
/*TODO njh create intergrate all POSTS to one tcp/ip connection */
published_this_pass++;
MS_DBG(F("pubDQTR1 delay"),delay_posted_pacing_ms ,F("mS : posted"), published_this_pass);
delay(delay_posted_pacing_ms);
}
} // while reading line
deszRdelClose(true);
Expand All @@ -892,12 +902,21 @@ void Logger::publishDataQuedToRemotes(bool internetPresent) {
deszQuedStart();
while ((dslStatus = deszQuedLine()) &&
cnt_for_pwr_analysis) {

/*At least one publish has been sucessfull.
* Slow Down sending based on publishers acceptance rate
* Each publish creates and tears down a TCP connection */
MS_DBG(F("pubDQTR2 delay"),delay_posted_pacing_ms ,F("mS : total posted"), published_this_pass);
delay(delay_posted_pacing_ms);

// setup for publisher to call deszqNextCh()
rspCode = dataPublishers[i]->publishData();
watchDogTimer.resetWatchDog();
postLogLine(i, rspCode);
if (HTTPSTATUS_CREATED_201 != rspCode) break;

tot_posted++;
published_this_pass++;

deszq_line[0] = 0; // Show completed

Expand Down Expand Up @@ -1408,7 +1427,7 @@ void Logger::postLogLine(uint8_t instance, int16_t rspParam) {
itoa(rspParam, tempBuffer, 10);
postsLogHndl.print(tempBuffer);
postsLogHndl.print(F(","));
itoa(dataPublishers[instance]->getTimerPost_mS(), tempBuffer, 10);
itoa(dataPublishers[instance]->getTimerPostTimeout_mS(), tempBuffer, 10);
postsLogHndl.print(tempBuffer);
postsLogHndl.print(F(","));
postsLogHndl.print(deszq_line);
Expand Down
28 changes: 18 additions & 10 deletions src/dataPublisherBase.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -352,20 +352,28 @@ class dataPublisher {
return useQueDataSource; // Default for not supported.
}

#if !defined TIMER_MMW_POST_TIMEOUT_DEF_MSEC
#define TIMER_MMW_POST_TIMEOUT_DEF_MSEC 10000L
#endif // TIMER_MMW_POST_TIMEOUT_DEF_MSEC
uint16_t _timerPostTimeout_ms = TIMER_MMW_POST_TIMEOUT_DEF_MSEC;
//Required to implement the following
uint16_t virtual setTimerPostTimeout_mS(uint16_t tpt_ms) {
MS_DBG(F("sTPT rejected "), tpt_ms);
return _timerPostTimeout_ms; // Default not updated.
MS_DBG(F("setTPT rejected "));
return 0; // Default not updated.
}

uint16_t _timerPost_ms = 0;
uint16_t virtual getTimerPost_mS() {
MS_DBG(F("gTP check"), _timerPost_ms);
return _timerPost_ms;
uint16_t virtual getTimerPostTimeout_mS() {
MS_DBG(F("getTPT rejected"));
return 0;
}

//Required to implement the following
uint16_t virtual setTimerPostPacing_mS(uint16_t tpt_ms) {
MS_DBG(F("setTPP rejected "));
return 0; // Default not updated.
}

uint16_t virtual getTimerPostPacing_mS() {
MS_DBG(F("setTPP rejected"));
return 0;
}

};

/* atl_extension */
Expand Down
7 changes: 5 additions & 2 deletions src/modems/DigiXBeeWifi.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,8 @@ bool DigiXBeeWifi::extraModemSetup(void) {
#endif // MS_DIGIXBEEWIFI_DEBUG
}
gsmModem.exitCommand();
} else {
success = false;
}

if (false == success) { PRINTOUT(F("Xbee '"), _modemName, F("' failed.")); }
Expand Down Expand Up @@ -435,7 +437,7 @@ bool DigiXBeeWifi::updateModemMetadata(void) {
if (0 == loggerModem::_pollModemMetaData) return false;

// Enter command mode only once for temp and battery
MS_DBG(F("Entering Command Mode:"));
MS_DBG(F("updateModemMetadata Entering Command Mode:"));
success &= gsmModem.commandMode();
if (POLL_MODEM_META_DATA_RSSI & loggerModem::_pollModemMetaData) {
// Assume a signal has already been established.
Expand Down Expand Up @@ -481,8 +483,9 @@ bool DigiXBeeWifi::updateModemMetadata(void) {
loggerModem::_priorModemTemp);
}
// Exit command modem
MS_DBG(F("Leaving Command Mode:"));
MS_DBG(F("updateModemMetadata Leaving Command Mode:"));
gsmModem.exitCommand();

++updateModemMetadata_cnt;
if (0 == rssi || (XBEE_RESET_THRESHOLD <= updateModemMetadata_cnt)) {
updateModemMetadata_cnt = 0;
Expand Down
Loading

0 comments on commit 936ecef

Please sign in to comment.