From 70d8a72bdf27cfff0afbd91992cadd43eca2c6ce Mon Sep 17 00:00:00 2001 From: Fabian Weisshaar Date: Wed, 10 Feb 2016 20:55:54 +0100 Subject: [PATCH 01/68] Add Steam In-Home Broadcast --- README | 4 +- lib/libprotoident.h | 1 + lib/proto_manager.cc | 1 + lib/udp/Makefile.am | 1 + lib/udp/lpi_steam_inhomebroadcast.cc | 69 ++++++++++++++++++++++++++++ lib/udp/udp_protocols.h | 1 + 6 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 lib/udp/lpi_steam_inhomebroadcast.cc diff --git a/README b/README index 0e790441..ec13e928 100644 --- a/README +++ b/README @@ -21,8 +21,8 @@ Authors: With contributions from: Donald Neal Aaron Murrihy - Paweł Foremski - elnappo + Paweł Foremski + Fabian Weisshaar Introduction ============ diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 525cd644..aa1cfbc9 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -283,6 +283,7 @@ typedef enum { LPI_PROTO_UDP_QUAKE, LPI_PROTO_UDP_STEAM, LPI_PROTO_UDP_STEAM_FRIENDS, + LPI_PROTO_UDP_STEAM_INHOMEBROADCAST, LPI_PROTO_UDP_WIN_MESSAGE, LPI_PROTO_UDP_GAMESPY, LPI_PROTO_UDP_EMULE, diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index b4e27128..78618b11 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -352,6 +352,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_starcraft(mod_map); register_steamfriends(mod_map); register_steam_localbroadcast(mod_map); + register_steam_inhomebroadcast(mod_map); register_steam_udp(mod_map); register_storm_worm(mod_map); register_stun(mod_map); diff --git a/lib/udp/Makefile.am b/lib/udp/Makefile.am index 490d4dc3..5bf1abb8 100644 --- a/lib/udp/Makefile.am +++ b/lib/udp/Makefile.am @@ -120,6 +120,7 @@ libprotoident_udp_la_SOURCES = \ lpi_starcraft.cc \ lpi_steam.cc \ lpi_steamfriends.cc \ + lpi_steam_inhomebroadcast.cc \ lpi_steam_localbroadcast.cc \ lpi_storm_worm.cc \ lpi_stun.cc \ diff --git a/lib/udp/lpi_steam_inhomebroadcast.cc b/lib/udp/lpi_steam_inhomebroadcast.cc new file mode 100644 index 00000000..c45d12a5 --- /dev/null +++ b/lib/udp/lpi_steam_inhomebroadcast.cc @@ -0,0 +1,69 @@ +/* + * This file is part of libprotoident + * + * Copyright (c) 2011 The University of Waikato, Hamilton, New Zealand. + * Author: Shane Alcock + * + * With contributions from: + * Aaron Murrihy + * Donald Neal + * + * All rights reserved. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with libprotoident; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * $Id$ + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_steam_inhomebroadcast_ports(uint16_t porta, uint16_t portb) { + if (porta == 27036 && portb == 27036) + return true; + return false; +} + +static inline bool match_steam_inhomebroadcast(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (!match_steam_inhomebroadcast_ports(data->server_port, data->client_port)) + return false; + + if (data->payload_len[0] == 0 || data->payload_len[1] == 0) { + if (match_str_both(data, "\xff\xff\xff\xff", "\x00\x00\x00\x00")) + return true; + } + + return false; +} + +static lpi_module_t lpi_steam_inhomebroadcast = { + LPI_PROTO_UDP_STEAM_INHOMEBROADCAST, + LPI_CATEGORY_GAMING, + "Steam_InHome_Broadcast", + 9, + match_steam_inhomebroadcast +}; + +void register_steam_inhomebroadcast(LPIModuleMap *mod_map) { + register_protocol(&lpi_steam_inhomebroadcast, mod_map); +} + diff --git a/lib/udp/udp_protocols.h b/lib/udp/udp_protocols.h index fdd5896c..c934ae37 100644 --- a/lib/udp/udp_protocols.h +++ b/lib/udp/udp_protocols.h @@ -121,6 +121,7 @@ void register_sql_worm(LPIModuleMap *mod_map); void register_ssdp(LPIModuleMap *mod_map); void register_starcraft(LPIModuleMap *mod_map); void register_steamfriends(LPIModuleMap *mod_map); +void register_steam_inhomebroadcast(LPIModuleMap *mod_map); void register_steam_localbroadcast(LPIModuleMap *mod_map); void register_steam_udp(LPIModuleMap *mod_map); void register_storm_worm(LPIModuleMap *mod_map); From 3da0f883d5373aef8bb268f64cab3f6af835f50d Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Thu, 12 Jan 2017 17:01:16 +1300 Subject: [PATCH 02/68] Update tools to use new libflowmanager API --- configure.ac | 2 +- tools/arff/lpi_arff.cc | 29 +++++++++++--------------- tools/find_unknown/lpi_find_unknown.cc | 27 +++++++++++------------- tools/protoident/lpi_protoident.cc | 29 +++++++++++--------------- 4 files changed, 37 insertions(+), 50 deletions(-) diff --git a/configure.ac b/configure.ac index c8a41cd8..afbdcbf2 100644 --- a/configure.ac +++ b/configure.ac @@ -50,7 +50,7 @@ AC_ARG_WITH(tools, AC_CHECK_LIB([trace], [trace_get_payload_length],,trace_found=0) if test "$build_tools" = yes; then - AC_CHECK_LIB([flowmanager], [lfm_release_flow],,lfm_found=0) + AC_CHECK_LIB([flowmanager], [lfm_version_three],,lfm_found=0) fi AC_CHECK_HEADERS([arpa/inet.h fcntl.h inttypes.h netdb.h netinet/in.h stdint.h stdlib.h string.h sys/socket.h sys/time.h syslog.h unistd.h]) diff --git a/tools/arff/lpi_arff.cc b/tools/arff/lpi_arff.cc index ac55fdee..c78ec2ff 100644 --- a/tools/arff/lpi_arff.cc +++ b/tools/arff/lpi_arff.cc @@ -91,6 +91,8 @@ static volatile int done = 0; char *local_mac = NULL; uint8_t mac_bytes[6]; +FlowManager *fm = NULL; + struct ident_stats { uint64_t pkts; uint64_t bytes; @@ -224,7 +226,7 @@ void expire_ident_flows(double ts, bool exp_flag) lpi_module_t *proto; /* Loop until libflowmanager has no more expired flows available */ - while ((expired = lfm_expire_next_flow(ts, exp_flag)) != NULL) { + while ((expired = fm->expireNextFlow(ts, exp_flag)) != NULL) { IdentFlow *ident = (IdentFlow *)expired->extension; @@ -232,9 +234,7 @@ void expire_ident_flows(double ts, bool exp_flag) /* Don't forget to free our custom data structure */ free(ident); - /* VERY IMPORTANT: delete the Flow structure itself, even - * though we did not directly allocate the memory ourselves */ - delete(expired); + fm->releaseFlow(expired); } } @@ -337,7 +337,7 @@ void per_packet(libtrace_packet_t *packet) /* Match the packet to a Flow - this will create a new flow if * there is no matching flow already in the Flow map and set the * is_new flag to true. */ - f = lfm_match_packet_to_flow(packet, dir, &is_new); + f = fm->matchPacketToFlow(packet, dir, &is_new); /* Libflowmanager did not like something about that packet - best to * just ignore it and carry on */ @@ -364,16 +364,8 @@ void per_packet(libtrace_packet_t *packet) * it needs from this packet */ lpi_update_data(packet, &ident->lpi, dir); - /* Update TCP state for TCP flows. The TCP state determines how long - * the flow can be idle before being expired by libflowmanager. For - * instance, flows for which we have only seen a SYN will expire much - * quicker than a TCP connection that has completed the handshake */ - if (tcp) { - lfm_check_tcp_flags(f, tcp, dir, ts); - } - /* Tell libflowmanager to update the expiry time for this flow */ - lfm_update_flow_expiry_timeout(f, ts); + fm->updateFlowExpiry(f, packet, dir, ts); } static void cleanup_signal(int sig) @@ -414,6 +406,8 @@ int main(int argc, char *argv[]) int dir; bool ignore_rfc1918 = false; + fm = new FlowManager(); + packet = trace_create_packet(); if (packet == NULL) { perror("Creating libtrace packet"); @@ -464,19 +458,19 @@ int main(int argc, char *argv[]) /* This tells libflowmanager to ignore any flows where an RFC1918 * private IP address is involved */ - if (lfm_set_config_option(LFM_CONFIG_IGNORE_RFC1918, &ignore_rfc1918) == 0) + if (fm->setConfigOption(LFM_CONFIG_IGNORE_RFC1918, &ignore_rfc1918) == 0) return -1; /* This tells libflowmanager not to replicate the TCP timewait * behaviour where closed TCP connections are retained in the Flow * map for an extra 2 minutes */ - if (lfm_set_config_option(LFM_CONFIG_TCP_TIMEWAIT, &opt_false) == 0) + if (fm->setConfigOption(LFM_CONFIG_TCP_TIMEWAIT, &opt_false) == 0) return -1; /* This tells libflowmanager not to utilise the fast expiry rules for * short-lived UDP connections - these rules are experimental * behaviour not in line with recommended "best" practice */ - if (lfm_set_config_option(LFM_CONFIG_SHORT_UDP, &opt_false) == 0) + if (fm->setConfigOption(LFM_CONFIG_SHORT_UDP, &opt_false) == 0) return -1; @@ -577,6 +571,7 @@ int main(int argc, char *argv[]) if (!done) expire_ident_flows(ts, true); lpi_free_library(); + delete(fm); return 0; } diff --git a/tools/find_unknown/lpi_find_unknown.cc b/tools/find_unknown/lpi_find_unknown.cc index b185c28e..a6a31acd 100644 --- a/tools/find_unknown/lpi_find_unknown.cc +++ b/tools/find_unknown/lpi_find_unknown.cc @@ -68,6 +68,8 @@ static volatile int done = 0; char *local_mac = NULL; uint8_t mac_bytes[6]; +FlowManager *fm = NULL; + /* This data structure is used to demonstrate how to use the 'extension' * pointer to store custom data for a flow */ typedef struct unknown { @@ -177,7 +179,7 @@ void expire_unknown_flows(double ts, bool exp_flag) { lpi_module_t *proto; /* Loop until libflowmanager has no more expired flows available */ - while ((expired = lfm_expire_next_flow(ts, exp_flag)) != NULL) { + while ((expired = fm->expireNextFlow(ts, exp_flag)) != NULL) { UnknownFlow *unk = (UnknownFlow *)expired->extension; @@ -193,7 +195,7 @@ void expire_unknown_flows(double ts, bool exp_flag) { /* VERY IMPORTANT: delete the Flow structure itself, even * though we did not directly allocate the memory ourselves */ - delete(expired); + fm->releaseFlow(expired); } } @@ -240,7 +242,7 @@ void per_packet(libtrace_packet_t *packet) { /* Match the packet to a Flow - this will create a new flow if * there is no matching flow already in the Flow map and set the * is_new flag to true. */ - f = lfm_match_packet_to_flow(packet, dir, &is_new); + f = fm->matchPacketToFlow(packet, dir, &is_new); /* Libflowmanager did not like something about that packet - best to * just ignore it and carry on */ @@ -275,16 +277,8 @@ void per_packet(libtrace_packet_t *packet) { * it needs from this packet */ lpi_update_data(packet, &unk->lpi, dir); - /* Update TCP state for TCP flows. The TCP state determines how long - * the flow can be idle before being expired by libflowmanager. For - * instance, flows for which we have only seen a SYN will expire much - * quicker than a TCP connection that has completed the handshake */ - if (tcp) { - lfm_check_tcp_flags(f, tcp, dir, ts); - } - /* Tell libflowmanager to update the expiry time for this flow */ - lfm_update_flow_expiry_timeout(f, ts); + fm->updateFlowExpiry(f, packet, dir, ts); } @@ -326,6 +320,8 @@ int main(int argc, char *argv[]) { int dir; bool ignore_rfc1918 = false; + fm = new FlowManager(); + packet = trace_create_packet(); if (packet == NULL) { perror("Creating libtrace packet"); @@ -380,20 +376,20 @@ int main(int argc, char *argv[]) { /* This tells libflowmanager to ignore any flows where an RFC1918 * private IP address is involved */ - if (lfm_set_config_option(LFM_CONFIG_IGNORE_RFC1918, + if (fm->setConfigOption(LFM_CONFIG_IGNORE_RFC1918, &ignore_rfc1918) == 0) return -1; /* This tells libflowmanager not to replicate the TCP timewait * behaviour where closed TCP connections are retained in the Flow * map for an extra 2 minutes */ - if (lfm_set_config_option(LFM_CONFIG_TCP_TIMEWAIT, &opt_false) == 0) + if (fm->setConfigOption(LFM_CONFIG_TCP_TIMEWAIT, &opt_false) == 0) return -1; /* This tells libflowmanager not to utilise the fast expiry rules for * short-lived UDP connections - these rules are experimental * behaviour not in line with recommended "best" practice */ - if (lfm_set_config_option(LFM_CONFIG_SHORT_UDP, &opt_false) == 0) + if (fm->setConfigOption(LFM_CONFIG_SHORT_UDP, &opt_false) == 0) return -1; @@ -463,6 +459,7 @@ int main(int argc, char *argv[]) { trace_destroy_packet(packet); expire_unknown_flows(ts, true); lpi_free_library(); + delete(fm); return 0; diff --git a/tools/protoident/lpi_protoident.cc b/tools/protoident/lpi_protoident.cc index 7c0d5331..81314442 100644 --- a/tools/protoident/lpi_protoident.cc +++ b/tools/protoident/lpi_protoident.cc @@ -68,6 +68,8 @@ static volatile int done = 0; char *local_mac = NULL; uint8_t mac_bytes[6]; +FlowManager *fm = NULL; + /* This data structure is used to demonstrate how to use the 'extension' * pointer to store custom data for a flow */ typedef struct ident { @@ -182,7 +184,7 @@ void expire_ident_flows(double ts, bool exp_flag) { lpi_module_t *proto; /* Loop until libflowmanager has no more expired flows available */ - while ((expired = lfm_expire_next_flow(ts, exp_flag)) != NULL) { + while ((expired = fm->expireNextFlow(ts, exp_flag)) != NULL) { IdentFlow *ident = (IdentFlow *)expired->extension; @@ -190,9 +192,7 @@ void expire_ident_flows(double ts, bool exp_flag) { /* Don't forget to free our custom data structure */ free(ident); - /* VERY IMPORTANT: delete the Flow structure itself, even - * though we did not directly allocate the memory ourselves */ - delete(expired); + fm->releaseFlow(expired); } } @@ -239,7 +239,7 @@ void per_packet(libtrace_packet_t *packet) { /* Match the packet to a Flow - this will create a new flow if * there is no matching flow already in the Flow map and set the * is_new flag to true. */ - f = lfm_match_packet_to_flow(packet, dir, &is_new); + f = fm->matchPacketToFlow(packet, dir, &is_new); /* Libflowmanager did not like something about that packet - best to * just ignore it and carry on */ @@ -276,17 +276,9 @@ void per_packet(libtrace_packet_t *packet) { * it needs from this packet */ lpi_update_data(packet, &ident->lpi, dir); - /* Update TCP state for TCP flows. The TCP state determines how long - * the flow can be idle before being expired by libflowmanager. For - * instance, flows for which we have only seen a SYN will expire much - * quicker than a TCP connection that has completed the handshake */ - if (tcp) { - lfm_check_tcp_flags(f, tcp, dir, ts); - } - assert(f); /* Tell libflowmanager to update the expiry time for this flow */ - lfm_update_flow_expiry_timeout(f, ts); + fm->updateFlowExpiry(f, packet, dir, ts); } @@ -328,6 +320,8 @@ int main(int argc, char *argv[]) { int dir; bool ignore_rfc1918 = false; + fm = new FlowManager(); + packet = trace_create_packet(); if (packet == NULL) { perror("Creating libtrace packet"); @@ -382,20 +376,20 @@ int main(int argc, char *argv[]) { /* This tells libflowmanager to ignore any flows where an RFC1918 * private IP address is involved */ - if (lfm_set_config_option(LFM_CONFIG_IGNORE_RFC1918, + if (fm->setConfigOption(LFM_CONFIG_IGNORE_RFC1918, &ignore_rfc1918) == 0) return -1; /* This tells libflowmanager not to replicate the TCP timewait * behaviour where closed TCP connections are retained in the Flow * map for an extra 2 minutes */ - if (lfm_set_config_option(LFM_CONFIG_TCP_TIMEWAIT, &opt_false) == 0) + if (fm->setConfigOption(LFM_CONFIG_TCP_TIMEWAIT, &opt_false) == 0) return -1; /* This tells libflowmanager not to utilise the fast expiry rules for * short-lived UDP connections - these rules are experimental * behaviour not in line with recommended "best" practice */ - if (lfm_set_config_option(LFM_CONFIG_SHORT_UDP, &opt_false) == 0) + if (fm->setConfigOption(LFM_CONFIG_SHORT_UDP, &opt_false) == 0) return -1; @@ -467,6 +461,7 @@ int main(int argc, char *argv[]) { if (!done) expire_ident_flows(ts, true); lpi_free_library(); + delete(fm); return 0; From ab9ba4f59c141432da41e584f6efedcf414432cb Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Wed, 18 Jan 2017 10:41:31 +1300 Subject: [PATCH 03/68] First attempt at a parallel lpi_protoident --- tools/protoident/lpi_protoident.cc | 336 +++++++++++++++++------------ 1 file changed, 204 insertions(+), 132 deletions(-) diff --git a/tools/protoident/lpi_protoident.cc b/tools/protoident/lpi_protoident.cc index 81314442..fda5c373 100644 --- a/tools/protoident/lpi_protoident.cc +++ b/tools/protoident/lpi_protoident.cc @@ -44,6 +44,7 @@ #include #include +#include #include #include @@ -55,20 +56,24 @@ enum { DIR_METHOD_PORT }; -int dir_method = DIR_METHOD_PORT; - -bool only_dir0 = false; -bool only_dir1 = false; - -bool require_both = false; -bool nat_hole = false; - +libtrace_t *currenttrace; static volatile int done = 0; -char *local_mac = NULL; -uint8_t mac_bytes[6]; +struct globalopts { + + int dir_method; + bool only_dir0 ; + bool only_dir1 ; + bool require_both ; + bool nat_hole ; + bool ignore_rfc1918 ; + char *local_mac ; + uint8_t mac_bytes[6]; +}; -FlowManager *fm = NULL; +struct threadlocal { + FlowManager *flowmanager; +}; /* This data structure is used to demonstrate how to use the 'extension' * pointer to store custom data for a flow */ @@ -83,6 +88,54 @@ typedef struct ident { lpi_data_t lpi; } IdentFlow; +static void *start_processing(libtrace_t *trace, libtrace_thread_t *thread, + void *global) { + + bool opt_true = true; + bool opt_false = false; + struct globalopts *opts = (struct globalopts *)global; + + struct threadlocal *tl = (struct threadlocal *)malloc(sizeof( + struct threadlocal)); + tl->flowmanager = new FlowManager(); + + /* This tells libflowmanager to ignore any flows where an RFC1918 + * private IP address is involved */ + if (tl->flowmanager->setConfigOption(LFM_CONFIG_IGNORE_RFC1918, + &(opts->ignore_rfc1918)) == 0) { + fprintf(stderr, "Failed to set IGNORE RFC 1918 option in libflowmanager\n"); + } + + /* This tells libflowmanager not to replicate the TCP timewait + * behaviour where closed TCP connections are retained in the Flow + * map for an extra 2 minutes */ + if (tl->flowmanager->setConfigOption(LFM_CONFIG_TCP_TIMEWAIT, + &opt_false) == 0) { + fprintf(stderr, "Failed to set TCP TIMEWAIT option in libflowmanager\n"); + } + + /* This tells libflowmanager not to utilise the fast expiry rules for + * short-lived UDP connections - these rules are experimental + * behaviour not in line with recommended "best" practice */ + if (tl->flowmanager->setConfigOption(LFM_CONFIG_SHORT_UDP, + &opt_false) == 0) { + fprintf(stderr, "Failed to set SHORT UDP option in libflowmanager\n"); + } + + return tl; +} + +static void *start_reporter(libtrace_t *trace, libtrace_thread_t *thread, + void *global) { + return NULL; +} + +static void stop_reporter(libtrace_t *trace, libtrace_thread_t *thread, + void *global, void *tls) { + if (tls) + free(tls); +} + /* Initialises the custom data for the given flow. Allocates memory for a * IdentFlow structure and ensures that the extension pointer points at * it. @@ -102,54 +155,55 @@ void init_ident_flow(Flow *f, uint8_t dir, double ts) { f->extension = ident; } -void dump_payload(lpi_data_t lpi, uint8_t dir) { +void dump_payload(lpi_data_t lpi, uint8_t dir, char *space, int spacelen) { int i; uint8_t *pl = (uint8_t *)(&(lpi.payload[dir])); uint32_t len = lpi.payload_len[dir]; + char ascii[4][5]; - printf("%08x ", ntohl(lpi.payload[dir])); - for (i = 0; i < 4; i++) { - if (*pl > 32 && *pl < 126) { - printf("%c", *pl); + snprintf(ascii[i], 5, "%c", *pl); } else { - printf("."); + snprintf(ascii[i], 5, ".", NULL); } pl ++; } - printf(" %u", lpi.payload_len[dir]); - - printf(" "); + snprintf(space, spacelen - 1, "%08x %s%s%s%s %u", + ntohl(lpi.payload[dir]), ascii[0], ascii[1], + ascii[2], ascii[3], lpi.payload_len[dir]); + } -void display_ident(Flow *f, IdentFlow *ident) { +char *display_ident(Flow *f, IdentFlow *ident, struct globalopts *opts) { - char s_ip[500]; - char c_ip[500]; - char str[1000]; + char s_ip[100]; + char c_ip[100]; + char pload_out[100]; + char pload_in[100]; + char *str = (char *)malloc(750); lpi_module_t *proto; - if (only_dir0 && ident->init_dir == 1) - return; - if (only_dir1 && ident->init_dir == 0) - return; - if (require_both) { + if (opts->only_dir0 && ident->init_dir == 1) + return NULL; + if (opts->only_dir1 && ident->init_dir == 0) + return NULL; + if (opts->require_both) { if (ident->lpi.payload_len[0] == 0 || ident->lpi.payload_len[1] == 0) { - return; + return NULL; } } - if (nat_hole) { + if (opts->nat_hole) { if (ident->init_dir != 1) - return; + return NULL; if (ident->lpi.payload_len[0] == 0 && ident->in_pkts <= 3) - return; + return NULL; } proto = lpi_guess_protocol(&ident->lpi); @@ -157,18 +211,19 @@ void display_ident(Flow *f, IdentFlow *ident) { f->id.get_server_ip_str(s_ip); f->id.get_client_ip_str(c_ip); - snprintf(str, 1000, "%s %s %s %u %u %u %.3f %.3f %" PRIu64 " %" PRIu64, + dump_payload(ident->lpi, 0, pload_out, 500); + dump_payload(ident->lpi, 1, pload_in, 500); + snprintf(str, 1000, "%s %s %s %u %u %u %.3f %.3f %" PRIu64 " %" PRIu64 "%s %s\n", proto->name, s_ip, c_ip, f->id.get_server_port(), f->id.get_client_port(), f->id.get_protocol(), ident->start_ts, ident->end_ts, - ident->out_bytes, ident->in_bytes); + ident->out_bytes, ident->in_bytes, + pload_out, pload_in); - printf("%s ", str); + return str; - dump_payload(ident->lpi, 0); - dump_payload(ident->lpi, 1); - printf("\n"); + //printf("\n"); } @@ -179,16 +234,23 @@ void display_ident(Flow *f, IdentFlow *ident) { * want the stats for all the still-active flows). Otherwise, only flows * that have been idle for longer than their expiry timeout will be expired. */ -void expire_ident_flows(double ts, bool exp_flag) { +void expire_ident_flows(libtrace_t *trace, libtrace_thread_t *thread, + struct globalopts *opts, FlowManager *fm, double ts, + bool exp_flag) { Flow *expired; lpi_module_t *proto; + char *result = NULL; /* Loop until libflowmanager has no more expired flows available */ while ((expired = fm->expireNextFlow(ts, exp_flag)) != NULL) { IdentFlow *ident = (IdentFlow *)expired->extension; - - display_ident(expired, ident); + result = display_ident(expired, ident, opts); + if (result) { + trace_publish_result(trace, thread, ident->end_ts, + (libtrace_generic_t){.ptr=result}, + RESULT_USER); + } /* Don't forget to free our custom data structure */ free(ident); @@ -196,8 +258,37 @@ void expire_ident_flows(double ts, bool exp_flag) { } } +static void stop_processing(libtrace_t *trace, libtrace_thread_t *thread, + void *global, void *tls) { + + struct globalopts *opts = (struct globalopts *)global; + struct threadlocal *tl = (struct threadlocal *)tls; -void per_packet(libtrace_packet_t *packet) { + expire_ident_flows(trace, thread, opts, tl->flowmanager, 0, true); + delete(tl->flowmanager); + free(tl); + + +} + + +static void per_result(libtrace_t *trace, libtrace_thread_t *sender, + void *global, void *tls, libtrace_result_t *result) { + + char *resultstr; + + if (result->type != RESULT_USER) + return; + + resultstr = (char *)result->value.ptr; + //printf("%s", resultstr); + free(resultstr); + +} + +static libtrace_packet_t *per_packet(libtrace_t *trace, + libtrace_thread_t *thread, void *global, void *tls, + libtrace_packet_t *packet) { Flow *f; IdentFlow *ident = NULL; @@ -209,42 +300,43 @@ void per_packet(libtrace_packet_t *packet) { double ts; uint16_t l3_type; + struct globalopts *opts = (struct globalopts *)global; + struct threadlocal *tl = (struct threadlocal *)tls; /* Libflowmanager only deals with IP traffic, so ignore anything * that does not have an IP header */ l3 = trace_get_layer3(packet, &l3_type, NULL); if (l3_type != TRACE_ETHERTYPE_IP && l3_type != TRACE_ETHERTYPE_IPV6) - return; - if (l3 == NULL) return; + return packet; + if (l3 == NULL) return packet; /* Expire all suitably idle flows */ ts = trace_get_seconds(packet); - expire_ident_flows(ts, false); + expire_ident_flows(trace, thread, opts, tl->flowmanager, ts, false); /* Determine packet direction */ - if (dir_method == DIR_METHOD_TRACE) { + if (opts->dir_method == DIR_METHOD_TRACE) { dir = trace_get_direction(packet); } - if (dir_method == DIR_METHOD_MAC) { - dir = mac_get_direction(packet, mac_bytes); + if (opts->dir_method == DIR_METHOD_MAC) { + dir = mac_get_direction(packet, opts->mac_bytes); } - if (dir_method == DIR_METHOD_PORT) { + if (opts->dir_method == DIR_METHOD_PORT) { dir = port_get_direction(packet); } - if (dir != 0 && dir != 1) - return; + return packet; /* Match the packet to a Flow - this will create a new flow if * there is no matching flow already in the Flow map and set the * is_new flag to true. */ - f = fm->matchPacketToFlow(packet, dir, &is_new); + f = tl->flowmanager->matchPacketToFlow(packet, dir, &is_new); /* Libflowmanager did not like something about that packet - best to * just ignore it and carry on */ if (f == NULL) { - return; + return packet; } tcp = trace_get_tcp(packet); @@ -278,14 +370,17 @@ void per_packet(libtrace_packet_t *packet) { assert(f); /* Tell libflowmanager to update the expiry time for this flow */ - fm->updateFlowExpiry(f, packet, dir, ts); - + tl->flowmanager->updateFlowExpiry(f, packet, dir, ts); + return packet; } static void cleanup_signal(int sig) { (void)sig; - done = 1; + if (!done) { + trace_pstop(currenttrace); + done = 1; + } } static void usage(char *prog) { @@ -306,55 +401,61 @@ static void usage(char *prog) { int main(int argc, char *argv[]) { - libtrace_t *trace; - libtrace_packet_t *packet; libtrace_filter_t *filter = NULL; struct sigaction sigact; - - bool opt_true = true; - bool opt_false = false; - + struct globalopts opts; int i, opt; double ts; char *filterstring = NULL; int dir; - bool ignore_rfc1918 = false; - fm = new FlowManager(); + libtrace_callback_set_t *processing, *reporter; - packet = trace_create_packet(); - if (packet == NULL) { - perror("Creating libtrace packet"); - return -1; - } + opts.dir_method = DIR_METHOD_PORT; + opts.only_dir0 = false; + opts.only_dir1 = false; + opts.require_both = false; + opts.nat_hole = false; + opts.ignore_rfc1918 = false; + opts.local_mac = NULL; + + processing = trace_create_callback_set(); + trace_set_starting_cb(processing, start_processing); + trace_set_stopping_cb(processing, stop_processing); + trace_set_packet_cb(processing, per_packet); + + reporter = trace_create_callback_set(); + trace_set_starting_cb(reporter, start_reporter); + trace_set_stopping_cb(reporter, stop_reporter); + trace_set_result_cb(reporter, per_result); while ((opt = getopt(argc, argv, "l:bHd:f:RhT")) != EOF) { switch (opt) { case 'l': - local_mac = optarg; - dir_method = DIR_METHOD_MAC; + opts.local_mac = optarg; + opts.dir_method = DIR_METHOD_MAC; break; case 'b': - require_both = true; + opts.require_both = true; break; case 'd': dir = atoi(optarg); if (dir == 0) - only_dir0 = true; + opts.only_dir0 = true; if (dir == 1) - only_dir1 = true; + opts.only_dir1 = true; break; case 'f': filterstring = optarg; break; case 'R': - ignore_rfc1918 = true; + opts.ignore_rfc1918 = true; break; case 'H': - nat_hole = true; + opts.nat_hole = true; break; case 'T': - dir_method = DIR_METHOD_TRACE; + opts.dir_method = DIR_METHOD_TRACE; break; case 'h': default: @@ -367,31 +468,12 @@ int main(int argc, char *argv[]) { filter = trace_create_filter(filterstring); } - if (local_mac != NULL) { - if (convert_mac_string(local_mac, mac_bytes) < 0) { - fprintf(stderr, "Invalid MAC: %s\n", local_mac); + if (opts.local_mac != NULL) { + if (convert_mac_string(opts.local_mac, opts.mac_bytes) < 0) { + fprintf(stderr, "Invalid MAC: %s\n", opts.local_mac); return 1; } - } - - /* This tells libflowmanager to ignore any flows where an RFC1918 - * private IP address is involved */ - if (fm->setConfigOption(LFM_CONFIG_IGNORE_RFC1918, - &ignore_rfc1918) == 0) - return -1; - - /* This tells libflowmanager not to replicate the TCP timewait - * behaviour where closed TCP connections are retained in the Flow - * map for an extra 2 minutes */ - if (fm->setConfigOption(LFM_CONFIG_TCP_TIMEWAIT, &opt_false) == 0) - return -1; - - /* This tells libflowmanager not to utilise the fast expiry rules for - * short-lived UDP connections - these rules are experimental - * behaviour not in line with recommended "best" practice */ - if (fm->setConfigOption(LFM_CONFIG_SHORT_UDP, &opt_false) == 0) - return -1; - + } sigact.sa_handler = cleanup_signal; sigemptyset(&sigact.sa_mask); @@ -407,61 +489,51 @@ int main(int argc, char *argv[]) { return -1; for (i = optind; i < argc; i++) { - + if (done) + break; fprintf(stderr, "%s\n", argv[i]); /* Bog-standard libtrace stuff for reading trace files */ - trace = trace_create(argv[i]); + currenttrace = trace_create(argv[i]); - if (!trace) { + if (!currenttrace) { perror("Creating libtrace trace"); return -1; } - if (trace_is_err(trace)) { - trace_perror(trace, "Opening trace file"); - trace_destroy(trace); + if (trace_is_err(currenttrace)) { + trace_perror(currenttrace, "Opening trace file"); + trace_destroy(currenttrace); continue; } - if (filter && trace_config(trace, TRACE_OPTION_FILTER, filter) == -1) { - trace_perror(trace, "Configuring filter"); - trace_destroy(trace); + if (filter && trace_config(currenttrace, TRACE_OPTION_FILTER, filter) == -1) { + trace_perror(currenttrace, "Configuring filter"); + trace_destroy(currenttrace); return -1; } - if (trace_start(trace) == -1) { - trace_perror(trace, "Starting trace"); - trace_destroy(trace); - continue; - } - while (trace_read_packet(trace, packet) > 0) { - ts = trace_get_seconds(packet); - per_packet(packet); - if (done) { - break; - } + trace_set_perpkt_threads(currenttrace, 3); - } + trace_set_combiner(currenttrace, &combiner_unordered, + (libtrace_generic_t){0}); - if (done) - break; + trace_set_hasher(currenttrace, HASHER_BIDIRECTIONAL, NULL, NULL); - if (trace_is_err(trace)) { - trace_perror(trace, "Reading packets"); - trace_destroy(trace); + if (trace_pstart(currenttrace, &opts, processing, reporter) == -1) { + trace_perror(currenttrace, "Starting trace"); + trace_destroy(currenttrace); continue; } - trace_destroy(trace); + trace_join(currenttrace); + trace_destroy(currenttrace); } - trace_destroy_packet(packet); - if (!done) - expire_ident_flows(ts, true); + trace_destroy_callback_set(processing); + trace_destroy_callback_set(reporter); lpi_free_library(); - delete(fm); return 0; From 4d5862d16552e5082eb77ea2306ef106b9d312be Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 10 Mar 2017 14:01:44 +1300 Subject: [PATCH 04/68] Fix uninitialised variable warning --- lib/tcp/lpi_git.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/tcp/lpi_git.cc b/lib/tcp/lpi_git.cc index 23e996a9..d3481b18 100644 --- a/lib/tcp/lpi_git.cc +++ b/lib/tcp/lpi_git.cc @@ -38,6 +38,7 @@ static inline bool match_git_header(uint32_t payload, uint32_t len) { if (len == 0) return true; + memset(headerstr, 0, 4); for (i = 0; i < 4; i++) { headerstr[i] = (char)(*pl); pl++; From d21d9cba2c3fa6af51824a1ffd059a04179ab7e6 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 10 Mar 2017 14:02:15 +1300 Subject: [PATCH 05/68] Add option to specify number of threads to use for processing Also re-enable output in lpi_protoident, as this had been disabled during testing :( --- tools/protoident/lpi_protoident.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tools/protoident/lpi_protoident.cc b/tools/protoident/lpi_protoident.cc index fda5c373..f20810d6 100644 --- a/tools/protoident/lpi_protoident.cc +++ b/tools/protoident/lpi_protoident.cc @@ -281,7 +281,7 @@ static void per_result(libtrace_t *trace, libtrace_thread_t *sender, return; resultstr = (char *)result->value.ptr; - //printf("%s", resultstr); + printf("%s", resultstr); free(resultstr); } @@ -386,7 +386,7 @@ static void cleanup_signal(int sig) { static void usage(char *prog) { printf("Usage details for %s\n\n", prog); - printf("%s [-l ] [-T] [-b] [-d ] [-f ] [-R] [-H] inputURI [inputURI ...]\n\n", prog); + printf("%s [-l ] [-T] [-b] [-d ] [-f ] [-R] [-H] [-t ] inputURI [inputURI ...]\n\n", prog); printf("Options:\n"); printf(" -l Determine direction based on representing the 'inside' \n portion of the network\n"); printf(" -T Use trace direction tags to determine direction\n"); @@ -395,6 +395,7 @@ static void usage(char *prog) { printf(" -f Ignore flows that do not match the given BPF filter\n"); printf(" -R Ignore flows involving private RFC 1918 address space\n"); printf(" -H Ignore flows that do not meet the criteria for an SPNAT hole\n"); + printf(" -t Share the workload over the given number of threads\n"); exit(0); } @@ -408,6 +409,7 @@ int main(int argc, char *argv[]) { double ts; char *filterstring = NULL; int dir; + int threads = 1; libtrace_callback_set_t *processing, *reporter; @@ -429,7 +431,7 @@ int main(int argc, char *argv[]) { trace_set_stopping_cb(reporter, stop_reporter); trace_set_result_cb(reporter, per_result); - while ((opt = getopt(argc, argv, "l:bHd:f:RhT")) != EOF) { + while ((opt = getopt(argc, argv, "l:bHd:f:RhTt:")) != EOF) { switch (opt) { case 'l': opts.local_mac = optarg; @@ -457,6 +459,11 @@ int main(int argc, char *argv[]) { case 'T': opts.dir_method = DIR_METHOD_TRACE; break; + case 't': + threads = atoi(optarg); + if (threads <= 0) + threads = 1; + break; case 'h': default: usage(argv[0]); @@ -513,7 +520,7 @@ int main(int argc, char *argv[]) { return -1; } - trace_set_perpkt_threads(currenttrace, 3); + trace_set_perpkt_threads(currenttrace, threads); trace_set_combiner(currenttrace, &combiner_unordered, (libtrace_generic_t){0}); From a4a6dd320292ba8e5b1086b2af5089d769bc45dc Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Mon, 13 Mar 2017 14:18:52 +1300 Subject: [PATCH 06/68] Fix bad length calculation in lpi_taobao.cc --- lib/tcp/lpi_taobao.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tcp/lpi_taobao.cc b/lib/tcp/lpi_taobao.cc index 91f68ce6..31bf1140 100644 --- a/lib/tcp/lpi_taobao.cc +++ b/lib/tcp/lpi_taobao.cc @@ -43,7 +43,7 @@ static inline bool match_taobao_req(uint32_t payload, uint32_t len) { /* Byte 4 is a length field, == len - 4 */ - uint32_t taolen = ntohl(payload & 0xffff); + uint32_t taolen = ntohl(payload) & 0xffff; if (MATCH(payload, 0xf5, 0x00, ANY, ANY)) { if (taolen == len - 4) @@ -61,7 +61,7 @@ static inline bool match_taobao_req(uint32_t payload, uint32_t len) { static inline bool match_taobao_resp(uint32_t payload, uint32_t len) { /* Byte 4 is a length field, == len - 4 */ - uint32_t taolen = ntohl(payload & 0xffff); + uint32_t taolen = ntohl(payload) & 0xffff; if (MATCH(payload, 0xf3, 0x00, ANY, ANY)) { if (taolen == len - 4) From 3c67d5a3fa59209320d7374bb550ce88f96ff932 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Mon, 13 Mar 2017 16:22:12 +1300 Subject: [PATCH 07/68] Match more PPstream combinations. --- lib/udp/lpi_ppstream.cc | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/lib/udp/lpi_ppstream.cc b/lib/udp/lpi_ppstream.cc index 9c48c63c..690cf6b8 100644 --- a/lib/udp/lpi_ppstream.cc +++ b/lib/udp/lpi_ppstream.cc @@ -100,13 +100,25 @@ static inline bool match_1580_ppstream(uint32_t payload, uint32_t len) { if (len == 24) { if (MATCH(payload, 0x15, 0x80, 0x51, 0x10)) return true; + if (MATCH(payload, 0x15, 0x80, 0x40, 0x01)) + return true; } return false; } +static inline bool match_1480_ppstream(uint32_t payload, uint32_t len) { + + if (len == 24) { + if (MATCH(payload, 0x14, 0x80, 0x50, 0x11)) + return true; + } + return false; + +} + static inline bool match_1b80_ppstream(uint32_t payload, uint32_t len) { if (len == 27) { - if (MATCH(payload, 0x1b, 0x80, 0x4e, 0x0c)) + if (MATCH(payload, 0x1b, 0x80, ANY, ANY)) return true; } return false; @@ -134,11 +146,23 @@ static inline bool match_ppstream(lpi_data_t *data, lpi_module_t *mod UNUSED) { if (match_1580_ppstream(data->payload[0], data->payload_len[0])) { if (match_1b80_ppstream(data->payload[1], data->payload_len[1])) return true; + if (match_1580_ppstream(data->payload[1], data->payload_len[1])) + return true; + if (match_1480_ppstream(data->payload[1], data->payload_len[1])) + return true; + if (match_any80_ppstream(data->payload[1])) + return true; } if (match_1580_ppstream(data->payload[1], data->payload_len[1])) { if (match_1b80_ppstream(data->payload[0], data->payload_len[0])) return true; + if (match_1580_ppstream(data->payload[0], data->payload_len[0])) + return true; + if (match_1480_ppstream(data->payload[0], data->payload_len[0])) + return true; + if (match_any80_ppstream(data->payload[0])) + return true; } if (match_8480_ppstream(data->payload[0], data->payload_len[0])) { From 25735b72223b283b111b927ef2e047914641cd1c Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Mon, 13 Mar 2017 16:23:01 +1300 Subject: [PATCH 08/68] Add another pattern for Facebook Zero --- lib/tcp/lpi_zero.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/tcp/lpi_zero.cc b/lib/tcp/lpi_zero.cc index 35402dfa..808f9677 100644 --- a/lib/tcp/lpi_zero.cc +++ b/lib/tcp/lpi_zero.cc @@ -55,6 +55,10 @@ static inline bool match_zero_fb_shlo(uint32_t payload, uint32_t len) { return true; if (MATCH(payload, 0x30, ANY, 0x0d, 0x00)) return true; + if (MATCH(payload, 0x30, 0x11, 0x0f, 0x00)) + return true; + if (MATCH(payload, 0x30, 0x30, 0x00, 0x00)) + return true; return false; } From b70a47281c7365cfd623dd9f94f9e9b9b88f17f9 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Mon, 13 Mar 2017 16:23:25 +1300 Subject: [PATCH 09/68] Allow matching of Shuijing 44 in both directions. --- lib/tcp/lpi_xunlei.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/tcp/lpi_xunlei.cc b/lib/tcp/lpi_xunlei.cc index 52741537..bd75c575 100644 --- a/lib/tcp/lpi_xunlei.cc +++ b/lib/tcp/lpi_xunlei.cc @@ -115,12 +115,16 @@ static inline bool match_xunlei(lpi_data_t *data, lpi_module_t *mod UNUSED) { return true; if (match_shuijing_46(data->payload[1], data->payload_len[1])) return true; + if (match_shuijing_44(data->payload[1], data->payload_len[1])) + return true; } if (match_shuijing_44(data->payload[1], data->payload_len[1])) { if (match_shuijing_3e(data->payload[0], data->payload_len[0])) return true; if (match_shuijing_46(data->payload[0], data->payload_len[0])) return true; + if (match_shuijing_44(data->payload[0], data->payload_len[0])) + return true; } From 25e83e7f7dfba7afd829b90061e87d49662a028e Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Thu, 16 Mar 2017 10:46:39 +1300 Subject: [PATCH 10/68] Don't leak memory when ignoring a flow in lpi_protoident --- tools/protoident/lpi_protoident.cc | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tools/protoident/lpi_protoident.cc b/tools/protoident/lpi_protoident.cc index f20810d6..349e5f64 100644 --- a/tools/protoident/lpi_protoident.cc +++ b/tools/protoident/lpi_protoident.cc @@ -185,7 +185,7 @@ char *display_ident(Flow *f, IdentFlow *ident, struct globalopts *opts) { char c_ip[100]; char pload_out[100]; char pload_in[100]; - char *str = (char *)malloc(750); + char *str; lpi_module_t *proto; if (opts->only_dir0 && ident->init_dir == 1) @@ -205,15 +205,16 @@ char *display_ident(Flow *f, IdentFlow *ident, struct globalopts *opts) { if (ident->lpi.payload_len[0] == 0 && ident->in_pkts <= 3) return NULL; } - + proto = lpi_guess_protocol(&ident->lpi); - + f->id.get_server_ip_str(s_ip); f->id.get_client_ip_str(c_ip); dump_payload(ident->lpi, 0, pload_out, 500); dump_payload(ident->lpi, 1, pload_in, 500); - snprintf(str, 1000, "%s %s %s %u %u %u %.3f %.3f %" PRIu64 " %" PRIu64 "%s %s\n", + str = (char *)malloc(750); + snprintf(str, 1000, "%s %s %s %u %u %u %.3f %.3f %" PRIu64 " %" PRIu64 "%s %s\n", proto->name, s_ip, c_ip, f->id.get_server_port(), f->id.get_client_port(), f->id.get_protocol(), ident->start_ts, @@ -222,10 +223,6 @@ char *display_ident(Flow *f, IdentFlow *ident, struct globalopts *opts) { pload_out, pload_in); return str; - - //printf("\n"); - - } /* Expires all flows that libflowmanager believes have been idle for too From 9fce92a83f49fb94b5bd2f0227e92a09acb4bb7c Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Thu, 16 Mar 2017 11:19:06 +1300 Subject: [PATCH 11/68] Fix missing space in lpi_protoident output --- tools/protoident/lpi_protoident.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/protoident/lpi_protoident.cc b/tools/protoident/lpi_protoident.cc index 349e5f64..35b47112 100644 --- a/tools/protoident/lpi_protoident.cc +++ b/tools/protoident/lpi_protoident.cc @@ -214,7 +214,7 @@ char *display_ident(Flow *f, IdentFlow *ident, struct globalopts *opts) { dump_payload(ident->lpi, 0, pload_out, 500); dump_payload(ident->lpi, 1, pload_in, 500); str = (char *)malloc(750); - snprintf(str, 1000, "%s %s %s %u %u %u %.3f %.3f %" PRIu64 " %" PRIu64 "%s %s\n", + snprintf(str, 1000, "%s %s %s %u %u %u %.3f %.3f %" PRIu64 " %" PRIu64 " %s %s\n", proto->name, s_ip, c_ip, f->id.get_server_port(), f->id.get_client_port(), f->id.get_protocol(), ident->start_ts, From 2abb21d184dca151f34e2fc0d4ea8cd53017dee6 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Thu, 16 Mar 2017 11:33:55 +1300 Subject: [PATCH 12/68] Replace lpi_find_unknown with parallel-compatible version --- tools/find_unknown/lpi_find_unknown.cc | 440 +++++++++++++++---------- 1 file changed, 262 insertions(+), 178 deletions(-) diff --git a/tools/find_unknown/lpi_find_unknown.cc b/tools/find_unknown/lpi_find_unknown.cc index a6a31acd..27193142 100644 --- a/tools/find_unknown/lpi_find_unknown.cc +++ b/tools/find_unknown/lpi_find_unknown.cc @@ -44,6 +44,7 @@ #include #include +#include #include #include @@ -55,116 +56,178 @@ enum { DIR_METHOD_PORT }; -int dir_method = DIR_METHOD_PORT; - -bool only_dir0 = false; -bool only_dir1 = false; - -bool require_both = false; -bool nat_hole = false; - +libtrace_t *currenttrace; static volatile int done = 0; -char *local_mac = NULL; -uint8_t mac_bytes[6]; +struct globalopts { -FlowManager *fm = NULL; + int dir_method; + bool only_dir0 ; + bool only_dir1 ; + bool require_both ; + bool nat_hole ; + bool ignore_rfc1918 ; + char *local_mac ; + uint8_t mac_bytes[6]; +}; + +struct threadlocal { + FlowManager *flowmanager; +}; /* This data structure is used to demonstrate how to use the 'extension' * pointer to store custom data for a flow */ -typedef struct unknown { +typedef struct ident { uint8_t init_dir; uint64_t in_bytes; uint64_t out_bytes; uint64_t in_pkts; uint64_t out_pkts; double start_ts; + double end_ts; lpi_data_t lpi; -} UnknownFlow; +} IdentFlow; + +static void *start_processing(libtrace_t *trace, libtrace_thread_t *thread, + void *global) { + + bool opt_true = true; + bool opt_false = false; + struct globalopts *opts = (struct globalopts *)global; + + struct threadlocal *tl = (struct threadlocal *)malloc(sizeof( + struct threadlocal)); + tl->flowmanager = new FlowManager(); + + /* This tells libflowmanager to ignore any flows where an RFC1918 + * private IP address is involved */ + if (tl->flowmanager->setConfigOption(LFM_CONFIG_IGNORE_RFC1918, + &(opts->ignore_rfc1918)) == 0) { + fprintf(stderr, "Failed to set IGNORE RFC 1918 option in libflowmanager\n"); + } + + /* This tells libflowmanager not to replicate the TCP timewait + * behaviour where closed TCP connections are retained in the Flow + * map for an extra 2 minutes */ + if (tl->flowmanager->setConfigOption(LFM_CONFIG_TCP_TIMEWAIT, + &opt_false) == 0) { + fprintf(stderr, "Failed to set TCP TIMEWAIT option in libflowmanager\n"); + } + + /* This tells libflowmanager not to utilise the fast expiry rules for + * short-lived UDP connections - these rules are experimental + * behaviour not in line with recommended "best" practice */ + if (tl->flowmanager->setConfigOption(LFM_CONFIG_SHORT_UDP, + &opt_false) == 0) { + fprintf(stderr, "Failed to set SHORT UDP option in libflowmanager\n"); + } + + return tl; +} + +static void *start_reporter(libtrace_t *trace, libtrace_thread_t *thread, + void *global) { + return NULL; +} + +static void stop_reporter(libtrace_t *trace, libtrace_thread_t *thread, + void *global, void *tls) { + if (tls) + free(tls); +} /* Initialises the custom data for the given flow. Allocates memory for a - * UnknownFlow structure and ensures that the extension pointer points at + * IdentFlow structure and ensures that the extension pointer points at * it. */ -void init_unknown_flow(Flow *f, uint8_t dir, double ts) { - UnknownFlow *unk = NULL; - - unk = (UnknownFlow *)malloc(sizeof(UnknownFlow)); - unk->init_dir = dir; - unk->in_bytes = 0; - unk->out_bytes = 0; - unk->in_pkts = 0; - unk->out_pkts = 0; - unk->start_ts = ts; - lpi_init_data(&unk->lpi); - f->extension = unk; +void init_ident_flow(Flow *f, uint8_t dir, double ts) { + IdentFlow *ident = NULL; + + ident = (IdentFlow *)malloc(sizeof(IdentFlow)); + ident->init_dir = dir; + ident->in_bytes = 0; + ident->out_bytes = 0; + ident->in_pkts = 0; + ident->out_pkts = 0; + ident->start_ts = ts; + ident->end_ts = ts; + lpi_init_data(&ident->lpi); + f->extension = ident; } -void dump_payload(lpi_data_t lpi, uint8_t dir) { +void dump_payload(lpi_data_t lpi, uint8_t dir, char *space, int spacelen) { int i; uint8_t *pl = (uint8_t *)(&(lpi.payload[dir])); uint32_t len = lpi.payload_len[dir]; + char ascii[4][5]; - printf("%08x ", ntohl(lpi.payload[dir])); - for (i = 0; i < 4; i++) { - if (*pl > 32 && *pl < 126) { - printf("%c", *pl); + snprintf(ascii[i], 5, "%c", *pl); } else { - printf("."); + snprintf(ascii[i], 5, ".", NULL); } pl ++; } - printf(" %u", lpi.payload_len[dir]); - - printf(" "); + snprintf(space, spacelen - 1, "%08x %s%s%s%s %u", + ntohl(lpi.payload[dir]), ascii[0], ascii[1], + ascii[2], ascii[3], lpi.payload_len[dir]); -} -void display_unknown(Flow *f, UnknownFlow *unk) { +} - char s_ip[500]; - char c_ip[500]; - char str[1000]; +char *display_ident(Flow *f, IdentFlow *ident, struct globalopts *opts) { + char s_ip[100]; + char c_ip[100]; + char pload_out[100]; + char pload_in[100]; + char *str; + lpi_module_t *proto; - if (only_dir0 && unk->init_dir == 1) - return; - if (only_dir1 && unk->init_dir == 0) - return; - if (require_both) { - if (unk->lpi.payload_len[0] == 0 || - unk->lpi.payload_len[1] == 0) { - return; + if (opts->only_dir0 && ident->init_dir == 1) + return NULL; + if (opts->only_dir1 && ident->init_dir == 0) + return NULL; + if (opts->require_both) { + if (ident->lpi.payload_len[0] == 0 || + ident->lpi.payload_len[1] == 0) { + return NULL; } } - if (nat_hole) { - if (unk->init_dir != 1) - return; - if (unk->lpi.payload_len[0] == 0 && unk->in_pkts <= 3) - return; + if (opts->nat_hole) { + if (ident->init_dir != 1) + return NULL; + if (ident->lpi.payload_len[0] == 0 && ident->in_pkts <= 3) + return NULL; + } + + proto = lpi_guess_protocol(&ident->lpi); + + if (proto->protocol != LPI_PROTO_UNKNOWN && + proto->protocol != LPI_PROTO_UDP) { + return NULL; } f->id.get_server_ip_str(s_ip); f->id.get_client_ip_str(c_ip); - - snprintf(str, 1000, "%s %s %u %u %u %.3f %" PRIu64 " %" PRIu64, + + dump_payload(ident->lpi, 0, pload_out, 500); + dump_payload(ident->lpi, 1, pload_in, 500); + str = (char *)malloc(750); + snprintf(str, 1000, "%s %s %u %u %u %.3f %.3f %" PRIu64 " %" PRIu64 " %s %s\n", s_ip, c_ip, f->id.get_server_port(), f->id.get_client_port(), - f->id.get_protocol(), unk->start_ts, - unk->out_bytes, unk->in_bytes); - - printf("%s ", str); - - dump_payload(unk->lpi, 0); - dump_payload(unk->lpi, 1); - printf("\n"); + f->id.get_protocol(), ident->start_ts, + ident->end_ts, + ident->out_bytes, ident->in_bytes, + pload_out, pload_in); + return str; } @@ -174,124 +237,159 @@ void display_unknown(Flow *f, UnknownFlow *unk) { * want the stats for all the still-active flows). Otherwise, only flows * that have been idle for longer than their expiry timeout will be expired. */ -void expire_unknown_flows(double ts, bool exp_flag) { +void expire_ident_flows(libtrace_t *trace, libtrace_thread_t *thread, + struct globalopts *opts, FlowManager *fm, double ts, + bool exp_flag) { Flow *expired; lpi_module_t *proto; + char *result = NULL; /* Loop until libflowmanager has no more expired flows available */ while ((expired = fm->expireNextFlow(ts, exp_flag)) != NULL) { - UnknownFlow *unk = (UnknownFlow *)expired->extension; - - proto = lpi_guess_protocol(&unk->lpi); - - if (proto->protocol == LPI_PROTO_UNKNOWN || - proto->protocol == LPI_PROTO_UDP) { - display_unknown(expired, unk); - } - + IdentFlow *ident = (IdentFlow *)expired->extension; + result = display_ident(expired, ident, opts); + if (result) { + trace_publish_result(trace, thread, ident->end_ts, + (libtrace_generic_t){.ptr=result}, + RESULT_USER); + } /* Don't forget to free our custom data structure */ - free(unk); + free(ident); - /* VERY IMPORTANT: delete the Flow structure itself, even - * though we did not directly allocate the memory ourselves */ fm->releaseFlow(expired); } } +static void stop_processing(libtrace_t *trace, libtrace_thread_t *thread, + void *global, void *tls) { -void per_packet(libtrace_packet_t *packet) { + struct globalopts *opts = (struct globalopts *)global; + struct threadlocal *tl = (struct threadlocal *)tls; + + expire_ident_flows(trace, thread, opts, tl->flowmanager, 0, true); + delete(tl->flowmanager); + free(tl); + + +} + + +static void per_result(libtrace_t *trace, libtrace_thread_t *sender, + void *global, void *tls, libtrace_result_t *result) { + + char *resultstr; + + if (result->type != RESULT_USER) + return; + + resultstr = (char *)result->value.ptr; + printf("%s", resultstr); + free(resultstr); + +} + +static libtrace_packet_t *per_packet(libtrace_t *trace, + libtrace_thread_t *thread, void *global, void *tls, + libtrace_packet_t *packet) { Flow *f; - UnknownFlow *unk = NULL; + IdentFlow *ident = NULL; uint8_t dir; bool is_new = false; libtrace_tcp_t *tcp = NULL; - double ts; + void *l3; + double ts; - void *l3; uint16_t l3_type; + struct globalopts *opts = (struct globalopts *)global; + struct threadlocal *tl = (struct threadlocal *)tls; /* Libflowmanager only deals with IP traffic, so ignore anything * that does not have an IP header */ l3 = trace_get_layer3(packet, &l3_type, NULL); if (l3_type != TRACE_ETHERTYPE_IP && l3_type != TRACE_ETHERTYPE_IPV6) - return; - if (l3 == NULL) return; + return packet; + if (l3 == NULL) return packet; /* Expire all suitably idle flows */ ts = trace_get_seconds(packet); - expire_unknown_flows(ts, false); + expire_ident_flows(trace, thread, opts, tl->flowmanager, ts, false); /* Determine packet direction */ - if (dir_method == DIR_METHOD_TRACE) { + if (opts->dir_method == DIR_METHOD_TRACE) { dir = trace_get_direction(packet); } - if (dir_method == DIR_METHOD_MAC) { - dir = mac_get_direction(packet, mac_bytes); + if (opts->dir_method == DIR_METHOD_MAC) { + dir = mac_get_direction(packet, opts->mac_bytes); } - if (dir_method == DIR_METHOD_PORT) { + if (opts->dir_method == DIR_METHOD_PORT) { dir = port_get_direction(packet); } - if (dir != 0 && dir != 1) - return; + return packet; /* Match the packet to a Flow - this will create a new flow if * there is no matching flow already in the Flow map and set the * is_new flag to true. */ - f = fm->matchPacketToFlow(packet, dir, &is_new); + f = tl->flowmanager->matchPacketToFlow(packet, dir, &is_new); /* Libflowmanager did not like something about that packet - best to * just ignore it and carry on */ if (f == NULL) { - return; + return packet; } tcp = trace_get_tcp(packet); /* If the returned flow is new, you will probably want to allocate and * initialise any custom data that you intend to track for the flow */ if (is_new) { - init_unknown_flow(f, dir, ts); - unk = (UnknownFlow *)f->extension; + init_ident_flow(f, dir, ts); + ident = (IdentFlow *)f->extension; } else { - unk = (UnknownFlow *)f->extension; + ident = (IdentFlow *)f->extension; if (tcp && tcp->syn && !tcp->ack) - unk->init_dir = dir; + ident->init_dir = dir; + if (ident->end_ts < ts) + ident->end_ts = ts; } /* Update our own byte and packet counters for reporting purposes */ if (dir == 0) { - unk->out_pkts += 1; - unk->out_bytes += trace_get_payload_length(packet); + ident->out_pkts += 1; + ident->out_bytes += trace_get_payload_length(packet); } else { - unk->in_bytes += trace_get_payload_length(packet); - unk->in_pkts += 1; + ident->in_bytes += trace_get_payload_length(packet); + ident->in_pkts += 1; } /* Pass the packet into libprotoident so it can extract any info * it needs from this packet */ - lpi_update_data(packet, &unk->lpi, dir); + lpi_update_data(packet, &ident->lpi, dir); + assert(f); /* Tell libflowmanager to update the expiry time for this flow */ - fm->updateFlowExpiry(f, packet, dir, ts); - + tl->flowmanager->updateFlowExpiry(f, packet, dir, ts); + return packet; } static void cleanup_signal(int sig) { (void)sig; - done = 1; + if (!done) { + trace_pstop(currenttrace); + done = 1; + } } static void usage(char *prog) { printf("Usage details for %s\n\n", prog); - printf("%s [-l ] [-T] [-b] [-d ] [-f ] [-R] [-H] inputURI [inputURI ...]\n\n", prog); + printf("%s [-l ] [-T] [-b] [-d ] [-f ] [-R] [-H] [-t ] inputURI [inputURI ...]\n\n", prog); printf("Options:\n"); printf(" -l Determine direction based on representing the 'inside' \n portion of the network\n"); printf(" -T Use trace direction tags to determine direction\n"); @@ -300,62 +398,75 @@ static void usage(char *prog) { printf(" -f Ignore flows that do not match the given BPF filter\n"); printf(" -R Ignore flows involving private RFC 1918 address space\n"); printf(" -H Ignore flows that do not meet the criteria for an SPNAT hole\n"); + printf(" -t Share the workload over the given number of threads\n"); exit(0); } int main(int argc, char *argv[]) { - libtrace_t *trace; - libtrace_packet_t *packet; libtrace_filter_t *filter = NULL; struct sigaction sigact; - - bool opt_true = true; - bool opt_false = false; - + struct globalopts opts; int i, opt; double ts; char *filterstring = NULL; int dir; - bool ignore_rfc1918 = false; + int threads = 1; - fm = new FlowManager(); + libtrace_callback_set_t *processing, *reporter; - packet = trace_create_packet(); - if (packet == NULL) { - perror("Creating libtrace packet"); - return -1; - } + opts.dir_method = DIR_METHOD_PORT; + opts.only_dir0 = false; + opts.only_dir1 = false; + opts.require_both = false; + opts.nat_hole = false; + opts.ignore_rfc1918 = false; + opts.local_mac = NULL; + + processing = trace_create_callback_set(); + trace_set_starting_cb(processing, start_processing); + trace_set_stopping_cb(processing, stop_processing); + trace_set_packet_cb(processing, per_packet); - while ((opt = getopt(argc, argv, "l:bHd:f:RhT")) != EOF) { + reporter = trace_create_callback_set(); + trace_set_starting_cb(reporter, start_reporter); + trace_set_stopping_cb(reporter, stop_reporter); + trace_set_result_cb(reporter, per_result); + + while ((opt = getopt(argc, argv, "l:bHd:f:RhTt:")) != EOF) { switch (opt) { case 'l': - local_mac = optarg; - dir_method = DIR_METHOD_MAC; + opts.local_mac = optarg; + opts.dir_method = DIR_METHOD_MAC; break; case 'b': - require_both = true; + opts.require_both = true; break; case 'd': dir = atoi(optarg); if (dir == 0) - only_dir0 = true; + opts.only_dir0 = true; if (dir == 1) - only_dir1 = true; + opts.only_dir1 = true; break; case 'f': filterstring = optarg; break; case 'R': - ignore_rfc1918 = true; + opts.ignore_rfc1918 = true; break; case 'H': - nat_hole = true; + opts.nat_hole = true; break; case 'T': - dir_method = DIR_METHOD_TRACE; + opts.dir_method = DIR_METHOD_TRACE; break; + case 't': + threads = atoi(optarg); + if (threads <= 0) + threads = 1; + break; case 'h': default: usage(argv[0]); @@ -367,31 +478,12 @@ int main(int argc, char *argv[]) { filter = trace_create_filter(filterstring); } - if (local_mac != NULL) { - if (convert_mac_string(local_mac, mac_bytes) < 0) { - fprintf(stderr, "Invalid MAC: %s\n", local_mac); + if (opts.local_mac != NULL) { + if (convert_mac_string(opts.local_mac, opts.mac_bytes) < 0) { + fprintf(stderr, "Invalid MAC: %s\n", opts.local_mac); return 1; } - } - - /* This tells libflowmanager to ignore any flows where an RFC1918 - * private IP address is involved */ - if (fm->setConfigOption(LFM_CONFIG_IGNORE_RFC1918, - &ignore_rfc1918) == 0) - return -1; - - /* This tells libflowmanager not to replicate the TCP timewait - * behaviour where closed TCP connections are retained in the Flow - * map for an extra 2 minutes */ - if (fm->setConfigOption(LFM_CONFIG_TCP_TIMEWAIT, &opt_false) == 0) - return -1; - - /* This tells libflowmanager not to utilise the fast expiry rules for - * short-lived UDP connections - these rules are experimental - * behaviour not in line with recommended "best" practice */ - if (fm->setConfigOption(LFM_CONFIG_SHORT_UDP, &opt_false) == 0) - return -1; - + } sigact.sa_handler = cleanup_signal; sigemptyset(&sigact.sa_mask); @@ -407,59 +499,51 @@ int main(int argc, char *argv[]) { return -1; for (i = optind; i < argc; i++) { - + if (done) + break; fprintf(stderr, "%s\n", argv[i]); /* Bog-standard libtrace stuff for reading trace files */ - trace = trace_create(argv[i]); + currenttrace = trace_create(argv[i]); - if (!trace) { + if (!currenttrace) { perror("Creating libtrace trace"); return -1; } - if (trace_is_err(trace)) { - trace_perror(trace, "Opening trace file"); - trace_destroy(trace); + if (trace_is_err(currenttrace)) { + trace_perror(currenttrace, "Opening trace file"); + trace_destroy(currenttrace); continue; } - if (filter && trace_config(trace, TRACE_OPTION_FILTER, filter) == -1) { - trace_perror(trace, "Configuring filter"); - trace_destroy(trace); + if (filter && trace_config(currenttrace, TRACE_OPTION_FILTER, filter) == -1) { + trace_perror(currenttrace, "Configuring filter"); + trace_destroy(currenttrace); return -1; } - if (trace_start(trace) == -1) { - trace_perror(trace, "Starting trace"); - trace_destroy(trace); - continue; - } - while (trace_read_packet(trace, packet) > 0) { - ts = trace_get_seconds(packet); - per_packet(packet); - if (done) - break; + trace_set_perpkt_threads(currenttrace, threads); - } + trace_set_combiner(currenttrace, &combiner_unordered, + (libtrace_generic_t){0}); - if (done) - break; + trace_set_hasher(currenttrace, HASHER_BIDIRECTIONAL, NULL, NULL); - if (trace_is_err(trace)) { - trace_perror(trace, "Reading packets"); - trace_destroy(trace); + if (trace_pstart(currenttrace, &opts, processing, reporter) == -1) { + trace_perror(currenttrace, "Starting trace"); + trace_destroy(currenttrace); continue; } - trace_destroy(trace); + trace_join(currenttrace); + trace_destroy(currenttrace); } - trace_destroy_packet(packet); - expire_unknown_flows(ts, true); + trace_destroy_callback_set(processing); + trace_destroy_callback_set(reporter); lpi_free_library(); - delete(fm); return 0; From 9d362e6b6a66bc939e02f8b345946d0e0ae5b2ef Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Thu, 16 Mar 2017 13:39:44 +1300 Subject: [PATCH 13/68] Update lpi_arff to be parallel-libtrace compatible --- tools/arff/lpi_arff.cc | 414 ++++++++++++++++++++++++++--------------- 1 file changed, 259 insertions(+), 155 deletions(-) diff --git a/tools/arff/lpi_arff.cc b/tools/arff/lpi_arff.cc index c78ec2ff..0153277e 100644 --- a/tools/arff/lpi_arff.cc +++ b/tools/arff/lpi_arff.cc @@ -68,6 +68,7 @@ #include #include +#include #include #include @@ -79,19 +80,24 @@ enum { DIR_METHOD_PORT }; -int dir_method = DIR_METHOD_PORT; - -bool only_dir0 = false; -bool only_dir1 = false; - -bool require_both = false; - +libtrace_t *currenttrace; static volatile int done = 0; -char *local_mac = NULL; -uint8_t mac_bytes[6]; +struct globalopts { -FlowManager *fm = NULL; + int dir_method; + bool only_dir0 ; + bool only_dir1 ; + bool require_both ; + bool nat_hole ; + bool ignore_rfc1918 ; + char *local_mac ; + uint8_t mac_bytes[6]; +}; + +struct threadlocal { + FlowManager *flowmanager; +}; struct ident_stats { uint64_t pkts; @@ -122,6 +128,57 @@ typedef struct ident { lpi_data_t lpi; } IdentFlow; + +static void *start_processing(libtrace_t *trace, libtrace_thread_t *thread, + void *global) { + + bool opt_true = true; + bool opt_false = false; + struct globalopts *opts = (struct globalopts *)global; + + struct threadlocal *tl = (struct threadlocal *)malloc(sizeof( + struct threadlocal)); + tl->flowmanager = new FlowManager(); + + /* This tells libflowmanager to ignore any flows where an RFC1918 + * private IP address is involved */ + if (tl->flowmanager->setConfigOption(LFM_CONFIG_IGNORE_RFC1918, + &(opts->ignore_rfc1918)) == 0) { + fprintf(stderr, "Failed to set IGNORE RFC 1918 option in libflowmanager\n"); + } + + /* This tells libflowmanager not to replicate the TCP timewait + * behaviour where closed TCP connections are retained in the Flow + * map for an extra 2 minutes */ + if (tl->flowmanager->setConfigOption(LFM_CONFIG_TCP_TIMEWAIT, + &opt_false) == 0) { + fprintf(stderr, "Failed to set TCP TIMEWAIT option in libflowmanager\n"); + } + + /* This tells libflowmanager not to utilise the fast expiry rules for + * short-lived UDP connections - these rules are experimental + * behaviour not in line with recommended "best" practice */ + if (tl->flowmanager->setConfigOption(LFM_CONFIG_SHORT_UDP, + &opt_false) == 0) { + fprintf(stderr, "Failed to set SHORT UDP option in libflowmanager\n"); + } + + return tl; +} + + +static void *start_reporter(libtrace_t *trace, libtrace_thread_t *thread, + void *global) { + return NULL; +} + +static void stop_reporter(libtrace_t *trace, libtrace_thread_t *thread, + void *global, void *tls) { + if (tls) + free(tls); +} + + /* Initialises the custom data for the given flow. Allocates memory for a * IdentFlow structure and ensures that the extension pointer points at * it. @@ -149,23 +206,50 @@ void init_ident_flow(Flow *f, uint8_t dir, double ts) f->extension = ident; } -void display_ident(Flow *f, IdentFlow *ident) +static void dump_len_stats(struct ident_stats *is, char *space, int spacelen) { + + if (is->pkts == 0) { + snprintf(space, spacelen - 1, ",0,0,0,0"); + } else { + snprintf(space, spacelen - 1, ",%u,%.0f,%u,%.0f", + is->pktlen_min, is->pktlen_mean, is->pktlen_max, + sqrt(is->pktlen_std / is->pkts)); + } +} + +static void dump_iat_stats(struct ident_stats *is, char *space, int spacelen) { + + if (is->pkts == 0) { + snprintf(space, spacelen - 1, ",0,0,0,0"); + } else { + snprintf(space, spacelen - 1, ",%u,%.0f,%u,%.0f", + is->iat_min, is->iat_mean, is->iat_max, + sqrt(is->iat_std / is->pkts)); + } +} + + +char *display_ident(Flow *f, IdentFlow *ident, struct globalopts *opts) { - char s_ip[500]; - char c_ip[500]; - char str[1000]; - lpi_module_t *proto; + char s_ip[100]; + char c_ip[100]; + char len_stats_out[200]; + char len_stats_in[200]; + char iat_stats_out[200]; + char iat_stats_in[200]; + char *str; + lpi_module_t *proto; struct ident_stats *is; int i; - if (only_dir0 && ident->init_dir == 1) - return; - if (only_dir1 && ident->init_dir == 0) - return; - if (require_both) { + if (opts->only_dir0 && ident->init_dir == 1) + return NULL; + if (opts->only_dir1 && ident->init_dir == 0) + return NULL; + if (opts->require_both) { if (ident->lpi.payload_len[0] == 0 || ident->lpi.payload_len[1] == 0) { - return; + return NULL; } } @@ -174,44 +258,25 @@ void display_ident(Flow *f, IdentFlow *ident) f->id.get_server_ip_str(s_ip); f->id.get_client_ip_str(c_ip); - /* basic statistics */ - printf("%s,%d,%" PRIu64 ",%" PRIu64 ",%" PRIu64 ",%" PRIu64, - proto->name, f->id.get_protocol(), - ident->out.pkts, ident->out.bytes, ident->in.pkts, ident->in.bytes); - - /* print packet length statistics */ - is = &ident->out; - for (i = 0; i < 2; i++) { - if (is->pkts == 0) { - printf(",0,0,0,0"); - } else { - printf(",%u,%.0f,%u,%.0f", - is->pktlen_min, is->pktlen_mean, is->pktlen_max, - sqrt(is->pktlen_std / is->pkts)); - } - is = &ident->in; - } - - /* print inter-arrival time statistics */ - is = &ident->out; - for (i = 0; i < 2; i++) { - if (is->pkts == 0) { - printf(",0,0,0,0"); - } else { - printf(",%u,%.0f,%u,%.0f", - is->iat_min, is->iat_mean, is->iat_max, - sqrt(is->iat_std / is->pkts)); - } - is = &ident->in; - } + str = (char *)malloc(750); - /* print total flow duration */ - printf(",%.0f", (ident->last_ts - ident->start_ts) * 1000000.0); + dump_len_stats(&ident->out, len_stats_out, 200); + dump_len_stats(&ident->in, len_stats_in, 200); + dump_iat_stats(&ident->out, iat_stats_out, 200); + dump_iat_stats(&ident->in, iat_stats_in, 200); - /* print flow start time */ - printf(",%f", ident->start_ts); + /* basic statistics */ + snprintf(str, 749, + "%s,%d,%" PRIu64 ",%" PRIu64 ",%" PRIu64 ",%" PRIu64 + "%s%s%s%s,%.0f,%f\n", + proto->name, f->id.get_protocol(), + ident->out.pkts, ident->out.bytes, ident->in.pkts, + ident->in.bytes, len_stats_out, len_stats_in, + iat_stats_out, iat_stats_in, + (ident->last_ts - ident->start_ts) * 1000000.0, + ident->start_ts); - printf("\n"); + return str; } /* Expires all flows that libflowmanager believes have been idle for too @@ -220,17 +285,25 @@ void display_ident(Flow *f, IdentFlow *ident) * want the stats for all the still-active flows). Otherwise, only flows * that have been idle for longer than their expiry timeout will be expired. */ -void expire_ident_flows(double ts, bool exp_flag) +void expire_ident_flows(libtrace_t *trace, libtrace_thread_t *thread, + struct globalopts *opts, FlowManager *fm, + double ts, bool exp_flag) { Flow *expired; lpi_module_t *proto; + char *result = NULL; /* Loop until libflowmanager has no more expired flows available */ while ((expired = fm->expireNextFlow(ts, exp_flag)) != NULL) { IdentFlow *ident = (IdentFlow *)expired->extension; - display_ident(expired, ident); + result = display_ident(expired, ident, opts); + if (result) { + trace_publish_result(trace, thread, ts, + (libtrace_generic_t){.ptr=result}, + RESULT_USER); + } /* Don't forget to free our custom data structure */ free(ident); @@ -238,6 +311,35 @@ void expire_ident_flows(double ts, bool exp_flag) } } +static void stop_processing(libtrace_t *trace, libtrace_thread_t *thread, + void *global, void *tls) { + + struct globalopts *opts = (struct globalopts *)global; + struct threadlocal *tl = (struct threadlocal *)tls; + + expire_ident_flows(trace, thread, opts, tl->flowmanager, 0, true); + delete(tl->flowmanager); + free(tl); + + +} + + +static void per_result(libtrace_t *trace, libtrace_thread_t *sender, + void *global, void *tls, libtrace_result_t *result) { + + char *resultstr; + + if (result->type != RESULT_USER) + return; + + resultstr = (char *)result->value.ptr; + printf("%s", resultstr); + free(resultstr); + +} + + /** Update flow statistics */ void per_packet_flow(libtrace_packet_t *packet, IdentFlow *ident, int dir, double ts) @@ -294,8 +396,9 @@ void per_packet_flow(libtrace_packet_t *packet, IdentFlow *ident, int dir, } /** This function receives each packet from libtrace */ -void per_packet(libtrace_packet_t *packet) -{ +static libtrace_packet_t *per_packet(libtrace_t *trace, + libtrace_thread_t *thread, void *global, void *tls, + libtrace_packet_t *packet) { Flow *f; IdentFlow *ident = NULL; uint8_t dir; @@ -306,43 +409,46 @@ void per_packet(libtrace_packet_t *packet) double ts; uint16_t l3_type; + struct globalopts *opts = (struct globalopts *)global; + struct threadlocal *tl = (struct threadlocal *)tls; + /* Libflowmanager only deals with IP traffic, so ignore anything * that does not have an IP header */ l3 = trace_get_layer3(packet, &l3_type, NULL); if (l3_type != TRACE_ETHERTYPE_IP && l3_type != TRACE_ETHERTYPE_IPV6) - return; - if (l3 == NULL) return; + return packet; + if (l3 == NULL) return packet; /* Expire all suitably idle flows */ ts = trace_get_seconds(packet); - expire_ident_flows(ts, false); + expire_ident_flows(trace, thread, opts, tl->flowmanager, ts, false); /* Determine packet direction * 0 is output, 1 is input */ - switch (dir_method) { + switch (opts->dir_method) { case DIR_METHOD_TRACE: dir = trace_get_direction(packet); break; case DIR_METHOD_MAC: - dir = mac_get_direction(packet, mac_bytes); + dir = mac_get_direction(packet, opts->mac_bytes); break; case DIR_METHOD_PORT: dir = port_get_direction(packet); } if (dir != 0 && dir != 1) - return; + return packet; /* Match the packet to a Flow - this will create a new flow if * there is no matching flow already in the Flow map and set the * is_new flag to true. */ - f = fm->matchPacketToFlow(packet, dir, &is_new); + f = tl->flowmanager->matchPacketToFlow(packet, dir, &is_new); /* Libflowmanager did not like something about that packet - best to * just ignore it and carry on */ if (f == NULL) { - return; + return packet; } tcp = trace_get_tcp(packet); @@ -365,19 +471,23 @@ void per_packet(libtrace_packet_t *packet) lpi_update_data(packet, &ident->lpi, dir); /* Tell libflowmanager to update the expiry time for this flow */ - fm->updateFlowExpiry(f, packet, dir, ts); + tl->flowmanager->updateFlowExpiry(f, packet, dir, ts); + return packet; } static void cleanup_signal(int sig) { (void)sig; - done = 1; + if (!done) { + done = 1; + trace_pstop(currenttrace); + } } static void usage(char *prog) { printf("Usage details for %s\n\n", prog); - printf("%s [-l ] [-T] [-b] [-d ] [-f ] [-R] inputURI [inputURI ...]\n\n", prog); + printf("%s [-l ] [-T] [-b] [-d ] [-f ] [-R] [-t ] inputURI [inputURI ...]\n\n", prog); printf("Options:\n"); printf(" -l Determine direction based on representing the 'inside'\n"); printf(" portion of the network\n"); @@ -387,6 +497,7 @@ static void usage(char *prog) printf(" direction\n"); printf(" -f Ignore flows that do not match the given BPF filter\n"); printf(" -R Ignore flows involving private RFC 1918 address space\n"); + printf(" -t Share the workload over the given number of threads\n"); exit(0); } @@ -395,7 +506,9 @@ int main(int argc, char *argv[]) libtrace_t *trace; libtrace_packet_t *packet; libtrace_filter_t *filter = NULL; - struct sigaction sigact; + struct sigaction sigact; + struct globalopts opts; + int threads = 1; bool opt_true = true; bool opt_false = false; @@ -406,39 +519,56 @@ int main(int argc, char *argv[]) int dir; bool ignore_rfc1918 = false; - fm = new FlowManager(); + libtrace_callback_set_t *processing, *reporter; - packet = trace_create_packet(); - if (packet == NULL) { - perror("Creating libtrace packet"); - return -1; - } + opts.dir_method = DIR_METHOD_PORT; + opts.only_dir0 = false; + opts.only_dir1 = false; + opts.require_both = false; + opts.nat_hole = false; + opts.ignore_rfc1918 = false; + opts.local_mac = NULL; - while ((opt = getopt(argc, argv, "l:bd:f:RhT")) != EOF) { + processing = trace_create_callback_set(); + trace_set_starting_cb(processing, start_processing); + trace_set_stopping_cb(processing, stop_processing); + trace_set_packet_cb(processing, per_packet); + + reporter = trace_create_callback_set(); + trace_set_starting_cb(reporter, start_reporter); + trace_set_stopping_cb(reporter, stop_reporter); + trace_set_result_cb(reporter, per_result); + + while ((opt = getopt(argc, argv, "l:bd:f:RhTt:")) != EOF) { switch (opt) { case 'l': - local_mac = optarg; - dir_method = DIR_METHOD_MAC; + opts.local_mac = optarg; + opts.dir_method = DIR_METHOD_MAC; break; case 'b': - require_both = true; + opts.require_both = true; break; case 'd': dir = atoi(optarg); if (dir == 0) - only_dir0 = true; + opts.only_dir0 = true; if (dir == 1) - only_dir1 = true; + opts.only_dir1 = true; break; case 'f': filterstring = optarg; break; case 'R': - ignore_rfc1918 = true; + opts.ignore_rfc1918 = true; break; case 'T': - dir_method = DIR_METHOD_TRACE; + opts.dir_method = DIR_METHOD_TRACE; break; + case 't': + threads = atoi(optarg); + if (threads <= 0) + threads = 1; + break; case 'h': default: usage(argv[0]); @@ -449,30 +579,12 @@ int main(int argc, char *argv[]) filter = trace_create_filter(filterstring); } - if (local_mac != NULL) { - if (convert_mac_string(local_mac, mac_bytes) < 0) { - fprintf(stderr, "Invalid MAC: %s\n", local_mac); - return 1; - } - } - - /* This tells libflowmanager to ignore any flows where an RFC1918 - * private IP address is involved */ - if (fm->setConfigOption(LFM_CONFIG_IGNORE_RFC1918, &ignore_rfc1918) == 0) - return -1; - - /* This tells libflowmanager not to replicate the TCP timewait - * behaviour where closed TCP connections are retained in the Flow - * map for an extra 2 minutes */ - if (fm->setConfigOption(LFM_CONFIG_TCP_TIMEWAIT, &opt_false) == 0) - return -1; - - /* This tells libflowmanager not to utilise the fast expiry rules for - * short-lived UDP connections - these rules are experimental - * behaviour not in line with recommended "best" practice */ - if (fm->setConfigOption(LFM_CONFIG_SHORT_UDP, &opt_false) == 0) - return -1; - + if (opts.local_mac != NULL) { + if (convert_mac_string(opts.local_mac, opts.mac_bytes) < 0) { + fprintf(stderr, "Invalid MAC: %s\n", opts.local_mac); + return 1; + } + } sigact.sa_handler = cleanup_signal; sigemptyset(&sigact.sa_mask); @@ -488,35 +600,9 @@ int main(int argc, char *argv[]) return -1; for (i = optind; i < argc; i++) { - + if (done) break; fprintf(stderr, "%s\n", argv[i]); - /* Bog-standard libtrace stuff for reading trace files */ - trace = trace_create(argv[i]); - - if (!trace) { - perror("Creating libtrace trace"); - return -1; - } - - if (trace_is_err(trace)) { - trace_perror(trace, "Opening trace file"); - trace_destroy(trace); - continue; - } - - if (filter && trace_config(trace, TRACE_OPTION_FILTER, filter) == -1) { - trace_perror(trace, "Configuring filter"); - trace_destroy(trace); - return -1; - } - - if (trace_start(trace) == -1) { - trace_perror(trace, "Starting trace"); - trace_destroy(trace); - continue; - } - /* printf arff file header */ printf("@relation '%s'\n", argv[i]); printf("\n"); @@ -547,31 +633,49 @@ int main(int argc, char *argv[]) printf("\n"); printf("@data\n"); - while (trace_read_packet(trace, packet) > 0) { - ts = trace_get_seconds(packet); - per_packet(packet); - if (done) { - break; - } - } + /* Bog-standard libtrace stuff for reading trace files */ + currenttrace = trace_create(argv[i]); - if (done) - break; + if (!currenttrace) { + perror("Creating libtrace trace"); + return -1; + } - if (trace_is_err(trace)) { - trace_perror(trace, "Reading packets"); - trace_destroy(trace); + if (trace_is_err(currenttrace)) { + trace_perror(currenttrace, "Opening trace file"); + trace_destroy(currenttrace); continue; } - trace_destroy(trace); + if (filter && trace_config(currenttrace, TRACE_OPTION_FILTER, filter) == -1) { + trace_perror(currenttrace, "Configuring filter"); + trace_destroy(currenttrace); + return -1; + } + + + trace_set_perpkt_threads(currenttrace, threads); + + trace_set_combiner(currenttrace, &combiner_unordered, + (libtrace_generic_t){0}); + + trace_set_hasher(currenttrace, HASHER_BIDIRECTIONAL, NULL, NULL); + + if (trace_pstart(currenttrace, &opts, processing, reporter) == -1) { + trace_perror(currenttrace, "Starting trace"); + trace_destroy(currenttrace); + continue; + } + + trace_join(currenttrace); + trace_destroy(currenttrace); + + } - trace_destroy_packet(packet); - if (!done) - expire_ident_flows(ts, true); - lpi_free_library(); - delete(fm); + trace_destroy_callback_set(processing); + trace_destroy_callback_set(reporter); + lpi_free_library(); return 0; } From 6ae518fb71e9f0fa9f12d7bebd5e6eda1f51f771 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 24 Mar 2017 15:17:29 +1300 Subject: [PATCH 14/68] New batch of protocol updates from Waikato campus data 3 new protocols: Glupteba Malware Tencent WNS Ark: Survival Evolved Improved protocols: 360 Safeguard Mongo Wechat WhatsApp --- lib/libprotoident.h | 3 ++ lib/proto_manager.cc | 3 ++ lib/tcp/Makefile.am | 2 + lib/tcp/lpi_360safeguard.cc | 15 +++++++ lib/tcp/lpi_glupteba.cc | 84 +++++++++++++++++++++++++++++++++++++ lib/tcp/lpi_mongo.cc | 24 +++++++++++ lib/tcp/lpi_wechat.cc | 6 ++- lib/tcp/lpi_whatsapp.cc | 2 +- lib/tcp/lpi_wns.cc | 55 ++++++++++++++++++++++++ lib/tcp/tcp_protocols.h | 2 + lib/udp/Makefile.am | 1 + lib/udp/lpi_ark.cc | 74 ++++++++++++++++++++++++++++++++ lib/udp/udp_protocols.h | 1 + 13 files changed, 270 insertions(+), 2 deletions(-) create mode 100644 lib/tcp/lpi_glupteba.cc create mode 100644 lib/tcp/lpi_wns.cc create mode 100644 lib/udp/lpi_ark.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 210d92df..ad43e9e1 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -316,6 +316,8 @@ typedef enum { LPI_PROTO_ZOOM, LPI_PROTO_S7COMM, LPI_PROTO_MAXICLOUD, + LPI_PROTO_GLUPTEBA, + LPI_PROTO_WNS, /* UDP Protocols */ LPI_PROTO_UDP, @@ -486,6 +488,7 @@ typedef enum { LPI_PROTO_UDP_YY, LPI_PROTO_UDP_OVERWATCH, LPI_PROTO_UDP_BACNET, + LPI_PROTO_UDP_ARK_SURVIVAL, /* Patterns that we can match, but do not know the protocol */ LPI_PROTO_REJECTION, /* All responses are 0x02 */ diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index 0c93be3a..3baad795 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -110,6 +110,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_funshion_tcp(mod_map); register_gamespy_tcp(mod_map); register_git(mod_map); + register_glupteba(mod_map); register_gnutella(mod_map); register_goku(mod_map); register_googlehangouts(mod_map); @@ -250,6 +251,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_whatsapp(mod_map); register_whois(mod_map); register_winmx(mod_map); + register_wns(mod_map); register_wow(mod_map); register_wuala(mod_map); register_xiami(mod_map); @@ -281,6 +283,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_amanda(mod_map); register_apple_facetime_init(mod_map); register_ares_udp(mod_map); + register_arksurvival(mod_map); register_arma_server(mod_map); register_arma3_server(mod_map); register_avast_secure_dns(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index e6078a88..8564bb61 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -43,6 +43,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_funshion.cc \ lpi_gamespy.cc \ lpi_git.cc \ + lpi_glupteba.cc \ lpi_gnutella.cc \ lpi_goku.cc \ lpi_googlehangouts.cc \ @@ -183,6 +184,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_whatsapp.cc \ lpi_whois.cc \ lpi_winmx.cc \ + lpi_wns.cc \ lpi_wow.cc \ lpi_wuala.cc \ lpi_xiami.cc \ diff --git a/lib/tcp/lpi_360safeguard.cc b/lib/tcp/lpi_360safeguard.cc index fee1e61d..dff22444 100644 --- a/lib/tcp/lpi_360safeguard.cc +++ b/lib/tcp/lpi_360safeguard.cc @@ -70,6 +70,14 @@ static inline bool match_360_03resp(uint32_t payload, uint32_t len) { } +static inline bool match_360_p2pupdate(uint32_t payload, uint32_t len) { + + if (len == 68 && MATCH(payload, 0x01, 0xaa, 0x4b, 0x79)) + return true; + return false; + +} + static inline bool match_360safeguard(lpi_data_t *data, lpi_module_t *mod UNUSED) { /* These patterns have been regularly seen on a machine with 360 @@ -98,6 +106,13 @@ static inline bool match_360safeguard(lpi_data_t *data, lpi_module_t *mod UNUSED return true; } + /* Have observed unencrypted traffic that appears to be downloading a + * .cab file to update 360 safeguard. */ + if (match_360_p2pupdate(data->payload[0], data->payload_len[0])) { + if (match_360_p2pupdate(data->payload[1], data->payload_len[1])) + return true; + } + return false; } diff --git a/lib/tcp/lpi_glupteba.cc b/lib/tcp/lpi_glupteba.cc new file mode 100644 index 00000000..8c2cde0d --- /dev/null +++ b/lib/tcp/lpi_glupteba.cc @@ -0,0 +1,84 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* https://www.trendmicro.com/vinfo/us/threat-encyclopedia/malware/bkdr_glupteba.yvg */ + +static inline bool match_glup_hello(uint32_t payload, uint32_t len) { + + if (len == 6 && MATCH(payload, 'H', 'E', 'L', 'L')) + return true; + return false; +} + +static inline bool match_glup_id(uint32_t payload, uint32_t len) { + + /* ID changes from infected host to infected host, but I'm + * going to assume the ID has a similar length. + */ + + if (len == 18 || len == 19) { + /* Always begins with @ */ + if (MATCH(payload, '@', ANY, ANY, ANY)) + return true; + } + return false; + +} + +static inline bool match_glupteba(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_glup_hello(data->payload[0], data->payload_len[0])) { + if (match_glup_id(data->payload[1], data->payload_len[1])) { + return true; + } + } + if (match_glup_hello(data->payload[1], data->payload_len[1])) { + if (match_glup_id(data->payload[0], data->payload_len[0])) { + return true; + } + } + + return false; +} + +static lpi_module_t lpi_glupteba = { + LPI_PROTO_GLUPTEBA, + LPI_CATEGORY_MALWARE, + "GluptebaBackdoor", + 50, + match_glupteba +}; + +void register_glupteba(LPIModuleMap *mod_map) { + register_protocol(&lpi_glupteba, mod_map); +} + diff --git a/lib/tcp/lpi_mongo.cc b/lib/tcp/lpi_mongo.cc index b2d5af72..939bb4dd 100644 --- a/lib/tcp/lpi_mongo.cc +++ b/lib/tcp/lpi_mongo.cc @@ -74,6 +74,20 @@ static inline bool match_mongo_short_reply(uint32_t payload, uint32_t len) { } +static inline bool match_mongo_short_req_0601(uint32_t payload, uint32_t len) { + if (len == 39 && MATCH(payload, 0x06, 0x01, 0x00, 0x00)) + return true; + return false; + +} + +static inline bool match_mongo_short_reply_c2(uint32_t payload, uint32_t len) { + if (len == 194 && MATCH(payload, 0xc2, 0x00, 0x00, 0x00)) + return true; + return false; + +} + static inline bool match_mongo(lpi_data_t *data, lpi_module_t *mod UNUSED) { /* Restrict to the default mongo port for now */ @@ -99,6 +113,16 @@ static inline bool match_mongo(lpi_data_t *data, lpi_module_t *mod UNUSED) { if (match_mongo_short_reply(data->payload[0], data->payload_len[0])) return true; } + + if (match_mongo_short_req_0601(data->payload[0], data->payload_len[0])) { + if (match_mongo_short_reply_c2(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_mongo_short_req_0601(data->payload[1], data->payload_len[1])) { + if (match_mongo_short_reply_c2(data->payload[0], data->payload_len[0])) + return true; + } return false; } diff --git a/lib/tcp/lpi_wechat.cc b/lib/tcp/lpi_wechat.cc index 23648691..be3fa29b 100644 --- a/lib/tcp/lpi_wechat.cc +++ b/lib/tcp/lpi_wechat.cc @@ -43,6 +43,8 @@ static inline bool match_wc_pair(uint32_t payloada, uint32_t lena, if (lena == 21 && MATCH(payloada, 0x00, 0x00, 0x00, 0x15)) { if (lenb == 25 && MATCH(payloadb, 0x00, 0x00, 0x00, 0x19)) return true; + if (lenb == 21 && MATCH(payloadb, 0x00, 0x00, 0x00, 0x15)) + return true; } return false; @@ -110,7 +112,7 @@ static inline bool match_wechat(lpi_data_t *data, lpi_module_t *mod UNUSED) { * This is not unique to WeChat though, so we need to be careful. */ - /* Only observed on port 80, 443 or 8080. Because the payload + /* Only observed on port 80, 443, 14000 or 8080. Because the payload * signature is not entirely unique to WeChat, let's restrict matches * to flows using those ports unless it shows up on other ports. */ @@ -120,6 +122,8 @@ static inline bool match_wechat(lpi_data_t *data, lpi_module_t *mod UNUSED) { valid_port = true; if (data->server_port == 443 || data->client_port == 443) valid_port = true; + if (data->server_port == 14000 || data->client_port == 14000) + valid_port = true; if (!valid_port) return false; diff --git a/lib/tcp/lpi_whatsapp.cc b/lib/tcp/lpi_whatsapp.cc index 9a32eee1..0431ed5c 100644 --- a/lib/tcp/lpi_whatsapp.cc +++ b/lib/tcp/lpi_whatsapp.cc @@ -58,7 +58,7 @@ static inline bool match_wa_second(uint32_t payload, uint32_t len) { } static inline bool match_wa_second_20(uint32_t payload, uint32_t len) { - uint32_t walen = ntohl(payload >> 8); + uint32_t walen = ntohl(payload) >> 8; if (len == 0) return true; diff --git a/lib/tcp/lpi_wns.cc b/lib/tcp/lpi_wns.cc new file mode 100644 index 00000000..f0d92c80 --- /dev/null +++ b/lib/tcp/lpi_wns.cc @@ -0,0 +1,55 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + + +static inline bool match_wns(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (MATCH(data->payload[0], 'w', 'n', 's', 0x00)) { + if (MATCH(data->payload[1], 'w', 'n', 's', 0x00)) + return true; + } + + return false; +} + +static lpi_module_t lpi_wns = { + LPI_PROTO_WNS, + LPI_CATEGORY_CLOUD, + "TencentWNS", + 8, + match_wns +}; + +void register_wns(LPIModuleMap *mod_map) { + register_protocol(&lpi_wns, mod_map); +} + diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index 00635fde..d55d6efe 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -71,6 +71,7 @@ void register_ftpdata(LPIModuleMap *mod_map); void register_funshion_tcp(LPIModuleMap *mod_map); void register_gamespy_tcp(LPIModuleMap *mod_map); void register_git(LPIModuleMap *mod_map); +void register_glupteba(LPIModuleMap *mod_map); void register_gnutella(LPIModuleMap *mod_map); void register_goku(LPIModuleMap *mod_map); void register_googlehangouts(LPIModuleMap *mod_map); @@ -211,6 +212,7 @@ void register_weibo(LPIModuleMap *mod_map); void register_whatsapp(LPIModuleMap *mod_map); void register_whois(LPIModuleMap *mod_map); void register_winmx(LPIModuleMap *mod_map); +void register_wns(LPIModuleMap *mod_map); void register_wow(LPIModuleMap *mod_map); void register_wuala(LPIModuleMap *mod_map); void register_xiami(LPIModuleMap *mod_map); diff --git a/lib/udp/Makefile.am b/lib/udp/Makefile.am index 2e3bdba9..1e1fe4d2 100644 --- a/lib/udp/Makefile.am +++ b/lib/udp/Makefile.am @@ -8,6 +8,7 @@ libprotoident_udp_la_SOURCES = \ lpi_amanda.cc \ lpi_apple_facetime_init.cc \ lpi_ares.cc \ + lpi_ark.cc \ lpi_arma_server.cc \ lpi_arma3_server.cc \ lpi_avast_secure_dns.cc \ diff --git a/lib/udp/lpi_ark.cc b/lib/udp/lpi_ark.cc new file mode 100644 index 00000000..bb130078 --- /dev/null +++ b/lib/udp/lpi_ark.cc @@ -0,0 +1,74 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + + +static inline bool match_ark_request(uint32_t payload, uint32_t len) { + if (len == 38 && MATCH(payload, 0x00, 0x80, 0x05, 0x00)) + return true; + return false; +} + +static inline bool match_ark_response(uint32_t payload, uint32_t len) { + if (len == 26 && MATCH(payload, 0x00, 0x00, 0x01, 0x00)) + return true; + return false; +} + + +static inline bool match_arksurvival(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + + if (match_ark_request(data->payload[0], data->payload_len[0])) { + if (match_ark_response(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_ark_request(data->payload[1], data->payload_len[1])) { + if (match_ark_response(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_arksurvival = { + LPI_PROTO_UDP_ARK_SURVIVAL, + LPI_CATEGORY_GAMING, + "ARKSurvivalEvolved", + 20, + match_arksurvival +}; + +void register_arksurvival(LPIModuleMap *mod_map) { + register_protocol(&lpi_arksurvival, mod_map); +} + diff --git a/lib/udp/udp_protocols.h b/lib/udp/udp_protocols.h index 4052c592..29efd3c9 100644 --- a/lib/udp/udp_protocols.h +++ b/lib/udp/udp_protocols.h @@ -38,6 +38,7 @@ void register_akamai_transfer(LPIModuleMap *mod_map); void register_amanda(LPIModuleMap *mod_map); void register_apple_facetime_init(LPIModuleMap *mod_map); void register_ares_udp(LPIModuleMap *mod_map); +void register_arksurvival(LPIModuleMap *mod_map); void register_arma_server(LPIModuleMap *mod_map); void register_arma3_server(LPIModuleMap *mod_map); void register_avast_secure_dns(LPIModuleMap *mod_map); From 546d5181b5cd7383391979485da8684e88a2d750 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Mon, 27 Mar 2017 14:51:57 +1300 Subject: [PATCH 15/68] Added 3 new protocols and improved 2 others New additions: * Panda.tv * 360 Safeguard P2P updates * Portmap RPC Improved: * Xunlei * Zoom --- lib/libprotoident.h | 3 ++ lib/proto_manager.cc | 3 ++ lib/tcp/Makefile.am | 1 + lib/tcp/lpi_pandatv.cc | 79 ++++++++++++++++++++++++++++++++++++++ lib/tcp/lpi_xunlei.cc | 10 +++++ lib/tcp/tcp_protocols.h | 1 + lib/udp/Makefile.am | 2 + lib/udp/lpi_360p2p.cc | 71 ++++++++++++++++++++++++++++++++++ lib/udp/lpi_portmap_rpc.cc | 59 ++++++++++++++++++++++++++++ lib/udp/lpi_zoom.cc | 9 +++++ lib/udp/udp_protocols.h | 2 + 11 files changed, 240 insertions(+) create mode 100644 lib/tcp/lpi_pandatv.cc create mode 100644 lib/udp/lpi_360p2p.cc create mode 100644 lib/udp/lpi_portmap_rpc.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index ad43e9e1..3d2a5080 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -318,6 +318,7 @@ typedef enum { LPI_PROTO_MAXICLOUD, LPI_PROTO_GLUPTEBA, LPI_PROTO_WNS, + LPI_PROTO_PANDATV, /* UDP Protocols */ LPI_PROTO_UDP, @@ -489,6 +490,8 @@ typedef enum { LPI_PROTO_UDP_OVERWATCH, LPI_PROTO_UDP_BACNET, LPI_PROTO_UDP_ARK_SURVIVAL, + LPI_PROTO_UDP_360P2P, + LPI_PROTO_UDP_PORTMAP_RPC, /* Patterns that we can match, but do not know the protocol */ LPI_PROTO_REJECTION, /* All responses are 0x02 */ diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index 3baad795..21d13e53 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -180,6 +180,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_openvpn(mod_map); register_palringo(mod_map); register_paltalk(mod_map); + register_pandatv(mod_map); register_pando(mod_map); register_pdbox(mod_map); register_pop3(mod_map); @@ -276,6 +277,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { int register_udp_protocols(LPIModuleMap *mod_map) { register_360cn(mod_map); + register_360p2p(mod_map); register_aachen_udp(mod_map); register_acercloud(mod_map); register_akamai(mod_map); @@ -388,6 +390,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_overwatch(mod_map); register_pando_udp(mod_map); register_planetside2(mod_map); + register_portmap_rpc(mod_map); register_pplive(mod_map); register_ppstream(mod_map); //register_probable_gnutella(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index 8564bb61..9b42d40f 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -113,6 +113,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_openvpn.cc \ lpi_palringo.cc \ lpi_paltalk.cc \ + lpi_pandatv.cc \ lpi_pando.cc \ lpi_pdbox.cc \ lpi_pop3.cc \ diff --git a/lib/tcp/lpi_pandatv.cc b/lib/tcp/lpi_pandatv.cc new file mode 100644 index 00000000..a424559a --- /dev/null +++ b/lib/tcp/lpi_pandatv.cc @@ -0,0 +1,79 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_panda_request(uint32_t payload, uint32_t len) { + + /* Seen length 156 and 160 so far, so could be variable length */ + if (MATCH(payload, 0x02, 0x00, 0x00, 0x00)) + return true; + return false; + +} + +static inline bool match_panda_reply(uint32_t payload, uint32_t len) { + + if (MATCH(payload, 0x01, 0x00, 0x00, 0x00) && len == 36) + return true; + return false; + +} + +static inline bool match_pandatv(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (data->server_port != 3105 && data->client_port != 3105) + return false; + + if (match_panda_request(data->payload[0], data->payload_len[0])) { + if (match_panda_reply(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_panda_request(data->payload[1], data->payload_len[1])) { + if (match_panda_reply(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_pandatv = { + LPI_PROTO_PANDATV, + LPI_CATEGORY_STREAMING, + "Panda.tv", + 140, + match_pandatv +}; + +void register_pandatv(LPIModuleMap *mod_map) { + register_protocol(&lpi_pandatv, mod_map); +} + diff --git a/lib/tcp/lpi_xunlei.cc b/lib/tcp/lpi_xunlei.cc index bd75c575..b001ba2e 100644 --- a/lib/tcp/lpi_xunlei.cc +++ b/lib/tcp/lpi_xunlei.cc @@ -44,6 +44,12 @@ static inline bool match_shuijing_3e(uint32_t payload, uint32_t len) { return false; } +static inline bool match_shuijing_41(uint32_t payload, uint32_t len) { + if (MATCH(payload, 0x41, 0x00, 0x00, 0x00)) + return true; + return false; +} + static inline bool match_shuijing_46(uint32_t payload, uint32_t len) { if (MATCH(payload, 0x46, 0x00, 0x00, 0x00)) return true; @@ -117,6 +123,8 @@ static inline bool match_xunlei(lpi_data_t *data, lpi_module_t *mod UNUSED) { return true; if (match_shuijing_44(data->payload[1], data->payload_len[1])) return true; + if (match_shuijing_41(data->payload[1], data->payload_len[1])) + return true; } if (match_shuijing_44(data->payload[1], data->payload_len[1])) { if (match_shuijing_3e(data->payload[0], data->payload_len[0])) @@ -125,6 +133,8 @@ static inline bool match_xunlei(lpi_data_t *data, lpi_module_t *mod UNUSED) { return true; if (match_shuijing_44(data->payload[0], data->payload_len[0])) return true; + if (match_shuijing_41(data->payload[0], data->payload_len[0])) + return true; } diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index d55d6efe..f6cd634a 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -141,6 +141,7 @@ void register_omegle(LPIModuleMap *mod_map); void register_openvpn(LPIModuleMap *mod_map); void register_palringo(LPIModuleMap *mod_map); void register_paltalk(LPIModuleMap *mod_map); +void register_pandatv(LPIModuleMap *mod_map); void register_pando(LPIModuleMap *mod_map); void register_pdbox(LPIModuleMap *mod_map); void register_pop3(LPIModuleMap *mod_map); diff --git a/lib/udp/Makefile.am b/lib/udp/Makefile.am index 1e1fe4d2..de5d0bc7 100644 --- a/lib/udp/Makefile.am +++ b/lib/udp/Makefile.am @@ -1,6 +1,7 @@ noinst_LTLIBRARIES = libprotoident_udp.la libprotoident_udp_la_SOURCES = \ lpi_360cn.cc \ + lpi_360p2p.cc \ lpi_aachen_udp.cc \ lpi_acercloud.cc \ lpi_akamai.cc \ @@ -114,6 +115,7 @@ libprotoident_udp_la_SOURCES = \ lpi_overwatch.cc \ lpi_pando.cc \ lpi_planetside2.cc \ + lpi_portmap_rpc.cc \ lpi_pplive.cc \ lpi_ppstream.cc \ lpi_probable_gnutella.cc \ diff --git a/lib/udp/lpi_360p2p.cc b/lib/udp/lpi_360p2p.cc new file mode 100644 index 00000000..4233fc28 --- /dev/null +++ b/lib/udp/lpi_360p2p.cc @@ -0,0 +1,71 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_360p2p_request(uint32_t payload, uint32_t len) { + if (len == 72 && MATCH(payload, 0x00, 0x00, 0x00, 0x00)) + return true; + return false; +} + +static inline bool match_360p2p_reply(uint32_t payload, uint32_t len) { + if (len == 40 && MATCH(payload, 0x00, 0x00, 0x00, 0x00)) + return true; + return false; +} + +static inline bool match_360p2p(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_360p2p_request(data->payload[0], data->payload_len[0])) { + if (match_360p2p_reply(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_360p2p_request(data->payload[1], data->payload_len[1])) { + if (match_360p2p_reply(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_360p2p = { + LPI_PROTO_UDP_360P2P, + LPI_CATEGORY_SECURITY, + "360Safeguard_P2P", + 211, + match_360p2p +}; + +void register_360p2p(LPIModuleMap *mod_map) { + register_protocol(&lpi_360p2p, mod_map); +} + diff --git a/lib/udp/lpi_portmap_rpc.cc b/lib/udp/lpi_portmap_rpc.cc new file mode 100644 index 00000000..563c315d --- /dev/null +++ b/lib/udp/lpi_portmap_rpc.cc @@ -0,0 +1,59 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + + +static inline bool match_portmap_rpc(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (data->server_port != 111 && data->client_port != 111) + return false; + + if (data->payload[0] != data->payload[1]) + return false; + + if (data->payload_len[0] == 46 || data->payload_len[1] == 46) + return true; + + return false; +} + +static lpi_module_t lpi_portmap_rpc = { + LPI_PROTO_UDP_PORTMAP_RPC, + LPI_CATEGORY_SERVICES, + "PortmapRPC", + 220, + match_portmap_rpc +}; + +void register_portmap_rpc(LPIModuleMap *mod_map) { + register_protocol(&lpi_portmap_rpc, mod_map); +} + diff --git a/lib/udp/lpi_zoom.cc b/lib/udp/lpi_zoom.cc index 618d3795..11efedb6 100644 --- a/lib/udp/lpi_zoom.cc +++ b/lib/udp/lpi_zoom.cc @@ -54,6 +54,13 @@ static inline bool match_zoom_02(uint32_t payload, uint32_t len) { } +static inline bool match_zoom_05(uint32_t payload) { + if (MATCH(payload, 0x05, 0x10, 0x01, 0x00)) + return true; + return false; + +} + static inline bool match_zoom(lpi_data_t *data, lpi_module_t *mod UNUSED) { if (data->server_port != 8801 && data->client_port != 8801) { @@ -74,6 +81,8 @@ static inline bool match_zoom(lpi_data_t *data, lpi_module_t *mod UNUSED) { return true; } + if (match_zoom_05(data->payload[0]) && match_zoom_05(data->payload[1])) + return true; return false; } diff --git a/lib/udp/udp_protocols.h b/lib/udp/udp_protocols.h index 29efd3c9..2686d06d 100644 --- a/lib/udp/udp_protocols.h +++ b/lib/udp/udp_protocols.h @@ -31,6 +31,7 @@ #include "proto_manager.h" void register_360cn(LPIModuleMap *mod_map); +void register_360p2p(LPIModuleMap *mod_map); void register_aachen_udp(LPIModuleMap *mod_map); void register_acercloud(LPIModuleMap *mod_map); void register_akamai(LPIModuleMap *mod_map); @@ -143,6 +144,7 @@ void register_orbit_udp(LPIModuleMap *mod_map); void register_overwatch(LPIModuleMap *mod_map); void register_pando_udp(LPIModuleMap *mod_map); void register_planetside2(LPIModuleMap *mod_map); +void register_portmap_rpc(LPIModuleMap *mod_map); void register_pplive(LPIModuleMap *mod_map); void register_ppstream(LPIModuleMap *mod_map); void register_probable_gnutella(LPIModuleMap *mod_map); From 6d877bbb5d75336905c3c3e8c243a1d74fd9059c Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 31 Mar 2017 14:19:46 +1300 Subject: [PATCH 16/68] Another batch of updates based on previously-unknown Waikato traffic New protocols: Destiny Facebook TURN QCloud Interactive Live Video Chivalry Nintendo Improved protocols: Mongo Glupteba QQ Live --- lib/Makefile.am | 1 + lib/libprotoident.h | 5 +++ lib/proto_manager.cc | 5 +++ lib/tcp/Makefile.am | 3 ++ lib/tcp/lpi_destiny.cc | 74 +++++++++++++++++++++++++++++++ lib/tcp/lpi_facebook_turn.cc | 84 +++++++++++++++++++++++++++++++++++ lib/tcp/lpi_glupteba.cc | 10 ++++- lib/tcp/lpi_mongo.cc | 85 ++++++++++-------------------------- lib/tcp/lpi_qcloud_ilvb.cc | 60 +++++++++++++++++++++++++ lib/tcp/tcp_protocols.h | 3 ++ lib/udp/Makefile.am | 2 + lib/udp/lpi_chivalry.cc | 59 +++++++++++++++++++++++++ lib/udp/lpi_nintendo.cc | 64 +++++++++++++++++++++++++++ lib/udp/lpi_qqlive.cc | 29 ++++++++++++ lib/udp/udp_protocols.h | 2 + 15 files changed, 422 insertions(+), 64 deletions(-) create mode 100644 lib/tcp/lpi_destiny.cc create mode 100644 lib/tcp/lpi_facebook_turn.cc create mode 100644 lib/tcp/lpi_qcloud_ilvb.cc create mode 100644 lib/udp/lpi_chivalry.cc create mode 100644 lib/udp/lpi_nintendo.cc diff --git a/lib/Makefile.am b/lib/Makefile.am index 822bc3c3..8f4ead54 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -11,3 +11,4 @@ INCLUDES=@ADD_INCLS@ libprotoident_la_LIBADD = @ADD_LIBS@ tcp/libprotoident_tcp.la \ udp/libprotoident_udp.la libprotoident_la_LDFLAGS = @ADD_LDFLAGS@ -version-info 2:10:0 +libprotoident_la_CPPFLAGS = -Werror diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 3d2a5080..ab73405b 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -319,6 +319,9 @@ typedef enum { LPI_PROTO_GLUPTEBA, LPI_PROTO_WNS, LPI_PROTO_PANDATV, + LPI_PROTO_FACEBOOK_TURN, + LPI_PROTO_DESTINY, + LPI_PROTO_QCLOUD_ILVB, /* UDP Protocols */ LPI_PROTO_UDP, @@ -492,6 +495,8 @@ typedef enum { LPI_PROTO_UDP_ARK_SURVIVAL, LPI_PROTO_UDP_360P2P, LPI_PROTO_UDP_PORTMAP_RPC, + LPI_PROTO_UDP_NINTENDO, + LPI_PROTO_UDP_CHIVALRY, /* Patterns that we can match, but do not know the protocol */ LPI_PROTO_REJECTION, /* All responses are 0x02 */ diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index 21d13e53..b53f19e8 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -91,6 +91,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_cryptic(mod_map); register_cvs(mod_map); register_dell_backup(mod_map); + register_destiny(mod_map); register_diablo3(mod_map); register_dianping_tcp(mod_map); register_directconnect(mod_map); @@ -101,6 +102,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_ea_games(mod_map); register_emule(mod_map); register_eye(mod_map); + register_facebook_turn(mod_map); register_fb_message(mod_map); register_ffxiv(mod_map); register_flash(mod_map); @@ -188,6 +190,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_postgresql(mod_map); register_pptp(mod_map); register_psn_store(mod_map); + register_qcloud_ilvb(mod_map); register_qq_tcp(mod_map); register_qqdownload(mod_map); register_qqlive_tcp(mod_map); @@ -299,6 +302,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_canon_mfnp(mod_map); register_callofduty(mod_map); register_checkpoint_rdp(mod_map); + register_chivalry(mod_map); register_cirn(mod_map); register_cisco_ipsec(mod_map); register_cisco_sslvpn(mod_map); @@ -378,6 +382,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_netcore_scan(mod_map); register_netflow(mod_map); register_newerth(mod_map); + register_nintendo(mod_map); register_noction(mod_map); register_noe(mod_map); register_norton(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index 9b42d40f..1cc316ec 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -24,6 +24,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_cryptic.cc \ lpi_cvs.cc \ lpi_dell_backup.cc \ + lpi_destiny.cc \ lpi_diablo3.cc \ lpi_dianping_tcp.cc \ lpi_directconnect.cc \ @@ -34,6 +35,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_ea_games.cc \ lpi_emule.cc \ lpi_eye.cc \ + lpi_facebook_turn.cc \ lpi_fb_message.cc \ lpi_ffxiv.cc \ lpi_flash.cc \ @@ -121,6 +123,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_postgresql.cc \ lpi_pptp.cc \ lpi_psn_store.cc \ + lpi_qcloud_ilvb.cc \ lpi_qq_tcp.cc \ lpi_qqdownload.cc \ lpi_qqlive_tcp.cc \ diff --git a/lib/tcp/lpi_destiny.cc b/lib/tcp/lpi_destiny.cc new file mode 100644 index 00000000..5066e08d --- /dev/null +++ b/lib/tcp/lpi_destiny.cc @@ -0,0 +1,74 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* Destiny -- multiplayer FPS by Bungie */ + +static inline bool match_destiny_request(uint32_t payload, uint32_t len) { + if (len == 140 && MATCH(payload, 0x01, 0x02, 0x00, 0x00)) + return true; + return false; +} + +static inline bool match_destiny_reply(uint32_t payload, uint32_t len) { + if (len == 142 && MATCH(payload, 0x01, 0x02, 0x00, 0x00)) + return true; + return false; +} + +static inline bool match_destiny(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_destiny_request(data->payload[0], data->payload_len[0])) { + if (match_destiny_reply(data->payload[1], data->payload_len[1])) + return true; + } + + + if (match_destiny_request(data->payload[1], data->payload_len[1])) { + if (match_destiny_reply(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_destiny = { + LPI_PROTO_DESTINY, + LPI_CATEGORY_GAMING, + "Destiny", + 23, + match_destiny +}; + +void register_destiny(LPIModuleMap *mod_map) { + register_protocol(&lpi_destiny, mod_map); +} + diff --git a/lib/tcp/lpi_facebook_turn.cc b/lib/tcp/lpi_facebook_turn.cc new file mode 100644 index 00000000..aac46e57 --- /dev/null +++ b/lib/tcp/lpi_facebook_turn.cc @@ -0,0 +1,84 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_fbturn_request(uint32_t payload, uint32_t len) { + /* 0x74 == len - 2, 0x0001 == binding request */ + + if (len == 118 && MATCH(payload, 0x00, 0x74, 0x00, 0x01)) + return true; + return false; +} + +static inline bool match_fbturn_reply(uint32_t payload, uint32_t len) { + /* 0x40 == len - 2, 0x0101 == binding accepted */ + + if (len == 66 && MATCH(payload, 0x00, 0x40, 0x01, 0x01)) + return true; + return false; +} + +static inline bool match_facebook_turn(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* Seems to be a slightly custom version of TURN, as there is a two + * byte length field preceding the conventional STUN header. Can't + * find any explanation for this in RFC 5766, so maybe it is a Facebook + * addition? + */ + + if (data->server_port != 443 && data->client_port != 443) + return false; + + if (match_fbturn_request(data->payload[0], data->payload_len[0])) { + if (match_fbturn_reply(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_fbturn_request(data->payload[1], data->payload_len[1])) { + if (match_fbturn_reply(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_facebook_turn = { + LPI_PROTO_FACEBOOK_TURN, + LPI_CATEGORY_NAT, + "FacebookTURN", + 55, + match_facebook_turn +}; + +void register_facebook_turn(LPIModuleMap *mod_map) { + register_protocol(&lpi_facebook_turn, mod_map); +} + diff --git a/lib/tcp/lpi_glupteba.cc b/lib/tcp/lpi_glupteba.cc index 8c2cde0d..33f0b544 100644 --- a/lib/tcp/lpi_glupteba.cc +++ b/lib/tcp/lpi_glupteba.cc @@ -34,7 +34,7 @@ static inline bool match_glup_hello(uint32_t payload, uint32_t len) { - if (len == 6 && MATCH(payload, 'H', 'E', 'L', 'L')) + if (MATCH(payload, 'H', 'E', 'L', 'L')) return true; return false; } @@ -45,7 +45,7 @@ static inline bool match_glup_id(uint32_t payload, uint32_t len) { * going to assume the ID has a similar length. */ - if (len == 18 || len == 19) { + if (len >= 18 && len <= 25) { /* Always begins with @ */ if (MATCH(payload, '@', ANY, ANY, ANY)) return true; @@ -60,11 +60,17 @@ static inline bool match_glupteba(lpi_data_t *data, lpi_module_t *mod UNUSED) { if (match_glup_id(data->payload[1], data->payload_len[1])) { return true; } + if (match_glup_hello(data->payload[1], data->payload_len[1])) { + return true; + } } if (match_glup_hello(data->payload[1], data->payload_len[1])) { if (match_glup_id(data->payload[0], data->payload_len[0])) { return true; } + if (match_glup_hello(data->payload[0], data->payload_len[0])) { + return true; + } } return false; diff --git a/lib/tcp/lpi_mongo.cc b/lib/tcp/lpi_mongo.cc index 939bb4dd..3b3456ce 100644 --- a/lib/tcp/lpi_mongo.cc +++ b/lib/tcp/lpi_mongo.cc @@ -31,59 +31,41 @@ #include "proto_common.h" /* Mongo protocol uses the first four bytes as a length field */ - static inline bool match_mongo_req(uint32_t payload, uint32_t len) { - if (len == 134 && MATCH(payload, 0x86, 0x00, 0x00, 0x00)) - return true; - if (len == 133 && MATCH(payload, 0x85, 0x00, 0x00, 0x00)) - return true; - if (len == 132 && MATCH(payload, 0x84, 0x00, 0x00, 0x00)) - return true; - - return false; -} - -static inline bool match_mongo_reply(uint32_t payload, uint32_t len) { - - if (len == 0) - return true; + uint32_t mongolen = ntohl(payload); - /* Only ever seen MTU sized packets as these responses */ - if (MATCH(payload, 0x84, 0x0c, 0x00, 0x00)) - return true; - if (MATCH(payload, 0x83, 0x0c, 0x00, 0x00)) - return true; - if (MATCH(payload, 0x82, 0x0c, 0x00, 0x00)) - return true; - - return false; -} + /* Most requests are very small */ + if (MATCH(payload, ANY, 0x00, 0x00, 0x00) || + MATCH(payload, ANY, 0x01, 0x00, 0x00)) { + /* Some mongo libraries manage to split requests across + * multiple packets, so we can't do a direct length + * match :( + */ + if (mongolen >= len) + return true; + } -static inline bool match_mongo_short_req(uint32_t payload, uint32_t len) { - if (len == 58 && MATCH(payload, 0x3a, 0x00, 0x00, 0x00)) - return true; return false; } -static inline bool match_mongo_short_reply(uint32_t payload, uint32_t len) { - if (len == 212 && MATCH(payload, 0xd4, 0x00, 0x00, 0x00)) - return true; - return false; +static inline bool match_mongo_reply(uint32_t payload, uint32_t len) { -} + uint32_t mongolen = ntohl(payload); -static inline bool match_mongo_short_req_0601(uint32_t payload, uint32_t len) { - if (len == 39 && MATCH(payload, 0x06, 0x01, 0x00, 0x00)) + /* If reply is short, mongolen should match the packet length */ + if (mongolen == len) return true; - return false; -} + /* If mongolen is large, packet len should be MTU-sized. Here, + * we'll assume an MTU of at least 1400 (sorry, people with + * tons of encapsulation) */ + if (mongolen > len) { + if (len >= 1400) + return true; + } -static inline bool match_mongo_short_reply_c2(uint32_t payload, uint32_t len) { - if (len == 194 && MATCH(payload, 0xc2, 0x00, 0x00, 0x00)) - return true; return false; } @@ -98,31 +80,10 @@ static inline bool match_mongo(lpi_data_t *data, lpi_module_t *mod UNUSED) { if (match_mongo_reply(data->payload[1], data->payload_len[1])) return true; } - if (match_mongo_req(data->payload[1], data->payload_len[1])) { if (match_mongo_reply(data->payload[0], data->payload_len[0])) return true; } - - if (match_mongo_short_req(data->payload[0], data->payload_len[0])) { - if (match_mongo_short_reply(data->payload[1], data->payload_len[1])) - return true; - } - - if (match_mongo_short_req(data->payload[1], data->payload_len[1])) { - if (match_mongo_short_reply(data->payload[0], data->payload_len[0])) - return true; - } - - if (match_mongo_short_req_0601(data->payload[0], data->payload_len[0])) { - if (match_mongo_short_reply_c2(data->payload[1], data->payload_len[1])) - return true; - } - - if (match_mongo_short_req_0601(data->payload[1], data->payload_len[1])) { - if (match_mongo_short_reply_c2(data->payload[0], data->payload_len[0])) - return true; - } return false; } @@ -130,7 +91,7 @@ static lpi_module_t lpi_mongo = { LPI_PROTO_MONGO, LPI_CATEGORY_DATABASES, "MongoDB", - 8, + 88, match_mongo }; diff --git a/lib/tcp/lpi_qcloud_ilvb.cc b/lib/tcp/lpi_qcloud_ilvb.cc new file mode 100644 index 00000000..be063ec5 --- /dev/null +++ b/lib/tcp/lpi_qcloud_ilvb.cc @@ -0,0 +1,60 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* Interactive Live Video Broadcasting, a service offered by Tencent QCloud. + * + * Seems to be some sort of SDK for developing live streaming applications. + */ + +static inline bool match_qcloud_ilvb(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* Packet sizes can vary -- 51,53,55,68 observed */ + if (MATCH(data->payload[0], 0x28, 0x00, 0x00, 0x00)) { + if (MATCH(data->payload[1], 0x28, 0x00, 0x00, 0x00)) + return true; + } + + return false; +} + +static lpi_module_t lpi_qcloud_ilvb = { + LPI_PROTO_QCLOUD_ILVB, + LPI_CATEGORY_STREAMING, + "QCloud_ILVB", + 199, + match_qcloud_ilvb +}; + +void register_qcloud_ilvb(LPIModuleMap *mod_map) { + register_protocol(&lpi_qcloud_ilvb, mod_map); +} + diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index f6cd634a..bae8f0c2 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -52,6 +52,7 @@ void register_crashplan(LPIModuleMap *mod_map); void register_cryptic(LPIModuleMap *mod_map); void register_cvs(LPIModuleMap *mod_map); void register_dell_backup(LPIModuleMap *mod_map); +void register_destiny(LPIModuleMap *mod_map); void register_diablo3(LPIModuleMap *mod_map); void register_dianping_tcp(LPIModuleMap *mod_map); void register_directconnect(LPIModuleMap *mod_map); @@ -62,6 +63,7 @@ void register_dxp(LPIModuleMap *mod_map); void register_ea_games(LPIModuleMap *mod_map); void register_emule(LPIModuleMap *mod_map); void register_eye(LPIModuleMap *mod_map); +void register_facebook_turn(LPIModuleMap *mod_map); void register_fb_message(LPIModuleMap *mod_map); void register_ffxiv(LPIModuleMap *mod_map); void register_flash(LPIModuleMap *mod_map); @@ -149,6 +151,7 @@ void register_pop3s(LPIModuleMap *mod_map); void register_postgresql(LPIModuleMap *mod_map); void register_pptp(LPIModuleMap *mod_map); void register_psn_store(LPIModuleMap *mod_map); +void register_qcloud_ilvb(LPIModuleMap *mod_map); void register_qq_tcp(LPIModuleMap *mod_map); void register_qqdownload(LPIModuleMap *mod_map); void register_qqlive_tcp(LPIModuleMap *mod_map); diff --git a/lib/udp/Makefile.am b/lib/udp/Makefile.am index de5d0bc7..fab922de 100644 --- a/lib/udp/Makefile.am +++ b/lib/udp/Makefile.am @@ -22,6 +22,7 @@ libprotoident_udp_la_SOURCES = \ lpi_btsync.cc \ lpi_canon_mfnp.cc \ lpi_checkpoint_rdp.cc \ + lpi_chivalry.cc \ lpi_cirn.cc \ lpi_cisco_ipsec.cc \ lpi_cisco_sslvpn.cc \ @@ -102,6 +103,7 @@ libprotoident_udp_la_SOURCES = \ lpi_netcore_scan.cc \ lpi_netflow.cc \ lpi_newerth.cc \ + lpi_nintendo.cc \ lpi_noction.cc \ lpi_noe.cc \ lpi_nopayload.cc \ diff --git a/lib/udp/lpi_chivalry.cc b/lib/udp/lpi_chivalry.cc new file mode 100644 index 00000000..7f29b27c --- /dev/null +++ b/lib/udp/lpi_chivalry.cc @@ -0,0 +1,59 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* Chivalry: Medieval Warfare -- a game */ + +static inline bool match_chivalry(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (MATCH(data->payload[0], 0x00, 0x00, 0x00, 0x00) && + MATCH(data->payload[1], 0x00, 0x00, 0x00, 0x00)) { + if (data->payload_len[0] == 12 && data->payload_len[1] == 15) + return true; + if (data->payload_len[1] == 12 && data->payload_len[0] == 15) + return true; + } + + return false; +} + +static lpi_module_t lpi_chivalry = { + LPI_PROTO_UDP_CHIVALRY, + LPI_CATEGORY_GAMING, + "Chivalry", + 200, + match_chivalry +}; + +void register_chivalry(LPIModuleMap *mod_map) { + register_protocol(&lpi_chivalry, mod_map); +} + diff --git a/lib/udp/lpi_nintendo.cc b/lib/udp/lpi_nintendo.cc new file mode 100644 index 00000000..5c6c9251 --- /dev/null +++ b/lib/udp/lpi_nintendo.cc @@ -0,0 +1,64 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + + +static inline bool match_nintendo_magic(uint32_t payload, uint32_t len) { + /* https://github.com/Shragei/SplatNet/blob/master/NinMainTiming.txt */ + + /* Only seen 64 byte packets so far */ + if (len == 64 && MATCH(payload, 0x32, 0xab, 0x98, 0x64)) + return true; + return false; +} + +static inline bool match_nintendo(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_nintendo_magic(data->payload[0], data->payload_len[0])) { + if (match_nintendo_magic(data->payload[1], data->payload_len[1])) + return true; + } + + return false; +} + +static lpi_module_t lpi_nintendo = { + LPI_PROTO_UDP_NINTENDO, + LPI_CATEGORY_GAMING, + "NintendoGames", + 5, + match_nintendo +}; + +void register_nintendo(LPIModuleMap *mod_map) { + register_protocol(&lpi_nintendo, mod_map); +} + diff --git a/lib/udp/lpi_qqlive.cc b/lib/udp/lpi_qqlive.cc index 256402eb..2a927870 100644 --- a/lib/udp/lpi_qqlive.cc +++ b/lib/udp/lpi_qqlive.cc @@ -31,6 +31,21 @@ #include "proto_manager.h" #include "proto_common.h" +static inline bool match_qqlive_p2p(uint32_t payload) { + + if (MATCH(payload, 0x1a, 0x10, 0x01, 0x20)) + return true; + return false; +} + +static inline bool match_qqlive_p2p_fe0a(uint32_t payload, uint32_t len) { + + if (MATCH(payload, 0xfe, 0x0a, 0x00, 0x00) && len == 13) + return true; + return false; + +} + static inline bool match_qqlive(lpi_data_t *data, lpi_module_t *mod UNUSED) { if (data->payload_len[0] == 0 || data->payload_len[1] == 0) { @@ -38,6 +53,20 @@ static inline bool match_qqlive(lpi_data_t *data, lpi_module_t *mod UNUSED) { return false; } + if (match_qqlive_p2p(data->payload[0])) { + if (match_qqlive_p2p(data->payload[1])) + return true; + if (match_qqlive_p2p_fe0a(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_qqlive_p2p(data->payload[1])) { + if (match_qqlive_p2p(data->payload[0])) + return true; + if (match_qqlive_p2p_fe0a(data->payload[0], data->payload_len[0])) + return true; + } + if (!match_qqlive_payload(data->payload[0], data->payload_len[0])) return false; if (!match_qqlive_payload(data->payload[1], data->payload_len[1])) diff --git a/lib/udp/udp_protocols.h b/lib/udp/udp_protocols.h index 2686d06d..4db49fa0 100644 --- a/lib/udp/udp_protocols.h +++ b/lib/udp/udp_protocols.h @@ -53,6 +53,7 @@ void register_btsync_udp(LPIModuleMap *mod_map); void register_callofduty(LPIModuleMap *mod_map); void register_canon_mfnp(LPIModuleMap *mod_map); void register_checkpoint_rdp(LPIModuleMap *mod_map); +void register_chivalry(LPIModuleMap *mod_map); void register_cirn(LPIModuleMap *mod_map); void register_cisco_ipsec(LPIModuleMap *mod_map); void register_cisco_sslvpn(LPIModuleMap *mod_map); @@ -132,6 +133,7 @@ void register_netbios_udp(LPIModuleMap *mod_map); void register_netcore_scan(LPIModuleMap *mod_map); void register_netflow(LPIModuleMap *mod_map); void register_newerth(LPIModuleMap *mod_map); +void register_nintendo(LPIModuleMap *mod_map); void register_noction(LPIModuleMap *mod_map); void register_noe(LPIModuleMap *mod_map); void register_norton(LPIModuleMap *mod_map); From 71fb180b5e62b5e397c88748ad5bc05969f03bc6 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Tue, 4 Apr 2017 10:22:49 +1200 Subject: [PATCH 17/68] Added 3 more new protocols * Doyo P2P * Bitcoin * LifeForge --- lib/libprotoident.h | 3 ++ lib/proto_manager.cc | 3 ++ lib/tcp/Makefile.am | 2 + lib/tcp/lpi_bitcoin.cc | 75 +++++++++++++++++++++++++++++++++++ lib/tcp/lpi_lifeforge.cc | 86 ++++++++++++++++++++++++++++++++++++++++ lib/tcp/tcp_protocols.h | 2 + lib/udp/Makefile.am | 1 + lib/udp/lpi_doyo.cc | 65 ++++++++++++++++++++++++++++++ lib/udp/udp_protocols.h | 1 + 9 files changed, 238 insertions(+) create mode 100644 lib/tcp/lpi_bitcoin.cc create mode 100644 lib/tcp/lpi_lifeforge.cc create mode 100644 lib/udp/lpi_doyo.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index ab73405b..4ac674e1 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -322,6 +322,8 @@ typedef enum { LPI_PROTO_FACEBOOK_TURN, LPI_PROTO_DESTINY, LPI_PROTO_QCLOUD_ILVB, + LPI_PROTO_BITCOIN, + LPI_PROTO_LIFEFORGE, /* UDP Protocols */ LPI_PROTO_UDP, @@ -497,6 +499,7 @@ typedef enum { LPI_PROTO_UDP_PORTMAP_RPC, LPI_PROTO_UDP_NINTENDO, LPI_PROTO_UDP_CHIVALRY, + LPI_PROTO_UDP_DOYO, /* Patterns that we can match, but do not know the protocol */ LPI_PROTO_REJECTION, /* All responses are 0x02 */ diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index b53f19e8..cd288a23 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -75,6 +75,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_apple_push(mod_map); register_ares(mod_map); register_badbaidu(mod_map); + register_bitcoin(mod_map); register_bitextend(mod_map); register_bittorrent(mod_map); register_blizzard(mod_map); @@ -147,6 +148,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_kik(mod_map); register_kuaibo(mod_map); register_ldap(mod_map); + register_lifeforge(mod_map); register_line(mod_map); register_llp2p(mod_map); register_maxicloud(mod_map); @@ -319,6 +321,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_discord(mod_map); register_dns_udp(mod_map); register_dota2(mod_map); + register_doyo(mod_map); register_driveshare(mod_map); register_dtls(mod_map); register_emule_udp(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index 1cc316ec..29a6e64e 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -8,6 +8,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_apple_push.cc \ lpi_ares.cc \ lpi_badbaidu.cc \ + lpi_bitcoin.cc \ lpi_bitextend.cc \ lpi_bittorrent.cc \ lpi_blizzard.cc \ @@ -80,6 +81,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_kik.cc \ lpi_kuaibo.cc \ lpi_ldap.cc \ + lpi_lifeforge.cc \ lpi_line.cc \ lpi_llp2p.cc \ lpi_maxicloud.cc \ diff --git a/lib/tcp/lpi_bitcoin.cc b/lib/tcp/lpi_bitcoin.cc new file mode 100644 index 00000000..c93f5b84 --- /dev/null +++ b/lib/tcp/lpi_bitcoin.cc @@ -0,0 +1,75 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + + +static inline bool match_bc_version(uint32_t payload, uint32_t len) { + + if (len == 24 && MATCH(payload, 0xf9, 0xbe, 0xb4, 0xd9)) + return true; + return false; +} + +static inline bool match_bc_version_reply(uint32_t payload, uint32_t len) { + + if (len >= 24 && MATCH(payload, 0xf9, 0xbe, 0xb4, 0xd9)) + return true; + return false; +} + +static inline bool match_bitcoin(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_bc_version(data->payload[0], data->payload_len[0])) { + if (match_bc_version_reply(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_bc_version(data->payload[1], data->payload_len[1])) { + if (match_bc_version_reply(data->payload[0], data->payload_len[0])) + return true; + } + + + return false; +} + +static lpi_module_t lpi_bitcoin = { + LPI_PROTO_BITCOIN, + LPI_CATEGORY_ECOMMERCE, + "Bitcoin", + 5, + match_bitcoin +}; + +void register_bitcoin(LPIModuleMap *mod_map) { + register_protocol(&lpi_bitcoin, mod_map); +} + diff --git a/lib/tcp/lpi_lifeforge.cc b/lib/tcp/lpi_lifeforge.cc new file mode 100644 index 00000000..be2eb563 --- /dev/null +++ b/lib/tcp/lpi_lifeforge.cc @@ -0,0 +1,86 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_lifeforge_login(uint32_t payload, uint32_t len) { + + uint32_t plen = ntohl(payload); + + /* I've only seen 0x25 in here, but that may vary depending on + * username length? */ + if (MATCH(payload, ANY, 0x00, 0x00, 0x00)) { + if (len == plen + 4) + return true; + } + + return false; +} + +static inline bool match_lifeforge_ping(uint32_t payload, uint32_t len) { + + if (MATCH(payload, 0x0e, 0x00, 0x00, 0x00)) { + if (len == 18) + return true; + if (len == 34) + return true; + } + + return false; + +} + +static inline bool match_lifeforge(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_lifeforge_login(data->payload[0], data->payload_len[0])) { + if (match_lifeforge_ping(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_lifeforge_login(data->payload[1], data->payload_len[1])) { + if (match_lifeforge_ping(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_lifeforge = { + LPI_PROTO_LIFEFORGE, + LPI_CATEGORY_GAMING, + "LifeForge", + 150, + match_lifeforge +}; + +void register_lifeforge(LPIModuleMap *mod_map) { + register_protocol(&lpi_lifeforge, mod_map); +} + diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index bae8f0c2..70e3aac7 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -36,6 +36,7 @@ void register_amp(LPIModuleMap *mod_map); void register_apple_push(LPIModuleMap *mod_map); void register_ares(LPIModuleMap *mod_map); void register_badbaidu(LPIModuleMap *mod_map); +void register_bitcoin(LPIModuleMap *mod_map); void register_bitextend(LPIModuleMap *mod_map); void register_bittorrent(LPIModuleMap *mod_map); void register_blizzard(LPIModuleMap *mod_map); @@ -108,6 +109,7 @@ void register_kaspersky(LPIModuleMap *mod_map); void register_kik(LPIModuleMap *mod_map); void register_kuaibo(LPIModuleMap *mod_map); void register_ldap(LPIModuleMap *mod_map); +void register_lifeforge(LPIModuleMap *mod_map); void register_line(LPIModuleMap *mod_map); void register_llp2p(LPIModuleMap *mod_map); void register_maxicloud(LPIModuleMap *mod_map); diff --git a/lib/udp/Makefile.am b/lib/udp/Makefile.am index fab922de..649e0d55 100644 --- a/lib/udp/Makefile.am +++ b/lib/udp/Makefile.am @@ -40,6 +40,7 @@ libprotoident_udp_la_SOURCES = \ lpi_discord.cc \ lpi_dns.cc \ lpi_dota2.cc \ + lpi_doyo.cc \ lpi_driveshare.cc \ lpi_dtls.cc \ lpi_emule.cc \ diff --git a/lib/udp/lpi_doyo.cc b/lib/udp/lpi_doyo.cc new file mode 100644 index 00000000..f7082652 --- /dev/null +++ b/lib/udp/lpi_doyo.cc @@ -0,0 +1,65 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* Doyo -- Chinese service for downloading free games (especially + * ones that you would otherwise have to buy!) + */ + +static inline bool match_doyo_p2p(uint32_t payload, uint32_t len) { + /* Not the strongest of rules, but not much we can do */ + if (len == 56 && MATCH(payload, 0x00, 0x00, 0x00, 0x00)) + return true; + return false; +} + +static inline bool match_doyo(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_doyo_p2p(data->payload[0], data->payload_len[0])) { + if (match_doyo_p2p(data->payload[1], data->payload_len[1])) + return true; + } + + return false; +} + +static lpi_module_t lpi_doyo = { + LPI_PROTO_UDP_DOYO, + LPI_CATEGORY_P2P, + "DoyoP2P", + 200, + match_doyo +}; + +void register_doyo(LPIModuleMap *mod_map) { + register_protocol(&lpi_doyo, mod_map); +} + diff --git a/lib/udp/udp_protocols.h b/lib/udp/udp_protocols.h index 4db49fa0..ec86b944 100644 --- a/lib/udp/udp_protocols.h +++ b/lib/udp/udp_protocols.h @@ -70,6 +70,7 @@ void register_directconnect_udp(LPIModuleMap *mod_map); void register_discord(LPIModuleMap *mod_map); void register_dns_udp(LPIModuleMap *mod_map); void register_dota2(LPIModuleMap *mod_map); +void register_doyo(LPIModuleMap *mod_map); void register_driveshare(LPIModuleMap *mod_map); void register_dtls(LPIModuleMap *mod_map); void register_emule_udp(LPIModuleMap *mod_map); From 40a64f386cca422d1d6edb2918aced0f966b0fa8 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Thu, 6 Apr 2017 11:02:37 +1200 Subject: [PATCH 18/68] Added 2 new protocols, improved one more Added: * Ace Stream * Netcat CCTV UDP Improved: * Facebook Zero --- lib/libprotoident.h | 2 ++ lib/proto_manager.cc | 2 ++ lib/tcp/Makefile.am | 1 + lib/tcp/lpi_acestream.cc | 62 ++++++++++++++++++++++++++++++++++ lib/tcp/lpi_zero.cc | 2 +- lib/tcp/tcp_protocols.h | 1 + lib/udp/Makefile.am | 1 + lib/udp/lpi_netcat_cctv_udp.cc | 62 ++++++++++++++++++++++++++++++++++ lib/udp/udp_protocols.h | 1 + 9 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 lib/tcp/lpi_acestream.cc create mode 100644 lib/udp/lpi_netcat_cctv_udp.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 4ac674e1..8507a2c1 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -324,6 +324,7 @@ typedef enum { LPI_PROTO_QCLOUD_ILVB, LPI_PROTO_BITCOIN, LPI_PROTO_LIFEFORGE, + LPI_PROTO_ACESTREAM, /* UDP Protocols */ LPI_PROTO_UDP, @@ -500,6 +501,7 @@ typedef enum { LPI_PROTO_UDP_NINTENDO, LPI_PROTO_UDP_CHIVALRY, LPI_PROTO_UDP_DOYO, + LPI_PROTO_UDP_NETCAT_CCTV, /* Patterns that we can match, but do not know the protocol */ LPI_PROTO_REJECTION, /* All responses are 0x02 */ diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index cd288a23..6ec2220b 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -68,6 +68,7 @@ void free_protocols(LPIModuleMap *mod_map) { int register_tcp_protocols(LPIModuleMap *mod_map) { register_360safeguard(mod_map); + register_acestream(mod_map); register_afp(mod_map); register_airdroid(mod_map); register_akamai_tcp(mod_map); @@ -382,6 +383,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_mystery_qq(mod_map); register_natpmp(mod_map); register_netbios_udp(mod_map); + register_netcat_cctv_udp(mod_map); register_netcore_scan(mod_map); register_netflow(mod_map); register_newerth(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index 29a6e64e..5d4cd60c 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -1,6 +1,7 @@ noinst_LTLIBRARIES = libprotoident_tcp.la libprotoident_tcp_la_SOURCES = \ lpi_360safeguard.cc \ + lpi_acestream.cc \ lpi_afp.cc \ lpi_airdroid.cc \ lpi_akamai.cc \ diff --git a/lib/tcp/lpi_acestream.cc b/lib/tcp/lpi_acestream.cc new file mode 100644 index 00000000..72daaafb --- /dev/null +++ b/lib/tcp/lpi_acestream.cc @@ -0,0 +1,62 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_ace_greet(uint32_t payload, uint32_t len) { + + if (len == 46 && MATCH(payload, 0x11, 'A', 'c', 'e')) + return true; + return false; + +} + +static inline bool match_acestream(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_ace_greet(data->payload[0], data->payload_len[0])) { + if (match_ace_greet(data->payload[1], data->payload_len[1])) + return true; + } + + return false; +} + +static lpi_module_t lpi_acestream = { + LPI_PROTO_ACESTREAM, + LPI_CATEGORY_P2P, + "AceStream", + 6, + match_acestream +}; + +void register_acestream(LPIModuleMap *mod_map) { + register_protocol(&lpi_acestream, mod_map); +} + diff --git a/lib/tcp/lpi_zero.cc b/lib/tcp/lpi_zero.cc index 808f9677..b9f8805c 100644 --- a/lib/tcp/lpi_zero.cc +++ b/lib/tcp/lpi_zero.cc @@ -57,7 +57,7 @@ static inline bool match_zero_fb_shlo(uint32_t payload, uint32_t len) { return true; if (MATCH(payload, 0x30, 0x11, 0x0f, 0x00)) return true; - if (MATCH(payload, 0x30, 0x30, 0x00, 0x00)) + if (MATCH(payload, 0x30, ANY, 0x00, 0x00)) return true; return false; } diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index 70e3aac7..8c1cfd54 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -29,6 +29,7 @@ #include "proto_manager.h" void register_360safeguard(LPIModuleMap *mod_map); +void register_acestream(LPIModuleMap *mod_map); void register_afp(LPIModuleMap *mod_map); void register_airdroid(LPIModuleMap *mod_map); void register_akamai_tcp(LPIModuleMap *mod_map); diff --git a/lib/udp/Makefile.am b/lib/udp/Makefile.am index 649e0d55..c7887374 100644 --- a/lib/udp/Makefile.am +++ b/lib/udp/Makefile.am @@ -101,6 +101,7 @@ libprotoident_udp_la_SOURCES = \ lpi_mystery_qq.cc \ lpi_natpmp.cc \ lpi_netbios.cc \ + lpi_netcat_cctv_udp.cc \ lpi_netcore_scan.cc \ lpi_netflow.cc \ lpi_newerth.cc \ diff --git a/lib/udp/lpi_netcat_cctv_udp.cc b/lib/udp/lpi_netcat_cctv_udp.cc new file mode 100644 index 00000000..53f7abb7 --- /dev/null +++ b/lib/udp/lpi_netcat_cctv_udp.cc @@ -0,0 +1,62 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_xmip_header(uint32_t payload, uint32_t len) { + if (len == 112 && MATCH(payload, 0x12, 0x20, 0xd0, 0x07)) + return true; + return false; +} + +static inline bool match_netcat_cctv_udp(lpi_data_t *data, + lpi_module_t *mod UNUSED) { + + + if (match_xmip_header(data->payload[0], data->payload_len[0])) { + if (match_xmip_header(data->payload[1], data->payload_len[1])) + return true; + } + + return false; +} + +static lpi_module_t lpi_netcat_cctv_udp = { + LPI_PROTO_UDP_NETCAT_CCTV, + LPI_CATEGORY_STREAMING, + "NetcatCCTV_UDP", + 22, + match_netcat_cctv_udp +}; + +void register_netcat_cctv_udp(LPIModuleMap *mod_map) { + register_protocol(&lpi_netcat_cctv_udp, mod_map); +} + diff --git a/lib/udp/udp_protocols.h b/lib/udp/udp_protocols.h index ec86b944..218243c3 100644 --- a/lib/udp/udp_protocols.h +++ b/lib/udp/udp_protocols.h @@ -131,6 +131,7 @@ void register_mystery_e9(LPIModuleMap *mod_map); void register_mystery_qq(LPIModuleMap *mod_map); void register_natpmp(LPIModuleMap *mod_map); void register_netbios_udp(LPIModuleMap *mod_map); +void register_netcat_cctv_udp(LPIModuleMap *mod_map); void register_netcore_scan(LPIModuleMap *mod_map); void register_netflow(LPIModuleMap *mod_map); void register_newerth(LPIModuleMap *mod_map); From 2bfdbdccd1ac1822fd28957cbd277917999464d4 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 7 Apr 2017 17:10:56 +1200 Subject: [PATCH 19/68] Added 3 new protocols New protocols: * N2Ping * Ubisoft Games * Gordon Ramsey Dash Also improved Facebook TURN to match some new patterns. --- lib/libprotoident.h | 3 ++ lib/proto_manager.cc | 3 ++ lib/tcp/lpi_facebook_turn.cc | 2 + lib/udp/Makefile.am | 3 ++ lib/udp/lpi_n2ping.cc | 74 +++++++++++++++++++++++++++++++++ lib/udp/lpi_ramseydash.cc | 79 ++++++++++++++++++++++++++++++++++++ lib/udp/lpi_ubisoft_games.cc | 66 ++++++++++++++++++++++++++++++ lib/udp/udp_protocols.h | 3 ++ 8 files changed, 233 insertions(+) create mode 100644 lib/udp/lpi_n2ping.cc create mode 100644 lib/udp/lpi_ramseydash.cc create mode 100644 lib/udp/lpi_ubisoft_games.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 8507a2c1..86381aff 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -502,6 +502,9 @@ typedef enum { LPI_PROTO_UDP_CHIVALRY, LPI_PROTO_UDP_DOYO, LPI_PROTO_UDP_NETCAT_CCTV, + LPI_PROTO_UDP_N2PING, + LPI_PROTO_UDP_RAMSEY_DASH, + LPI_PROTO_UDP_UBISOFT_GAMES, /* Patterns that we can match, but do not know the protocol */ LPI_PROTO_REJECTION, /* All responses are 0x02 */ diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index 6ec2220b..02ae4d96 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -381,6 +381,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_mystery_99(mod_map); register_mystery_e9(mod_map); register_mystery_qq(mod_map); + register_n2ping(mod_map); register_natpmp(mod_map); register_netbios_udp(mod_map); register_netcat_cctv_udp(mod_map); @@ -413,6 +414,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_quake(mod_map); register_quic(mod_map); register_radius(mod_map); + register_ramsey_dash(mod_map); register_real(mod_map); register_roblox(mod_map); register_robocraft(mod_map); @@ -447,6 +449,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_traceroute(mod_map); register_tremulous(mod_map); register_tvants(mod_map); + register_ubisoft_games(mod_map); register_udp_no_payload(mod_map); register_unreal(mod_map); register_ventrilo(mod_map); diff --git a/lib/tcp/lpi_facebook_turn.cc b/lib/tcp/lpi_facebook_turn.cc index aac46e57..1a42f073 100644 --- a/lib/tcp/lpi_facebook_turn.cc +++ b/lib/tcp/lpi_facebook_turn.cc @@ -35,6 +35,8 @@ static inline bool match_fbturn_request(uint32_t payload, uint32_t len) { if (len == 118 && MATCH(payload, 0x00, 0x74, 0x00, 0x01)) return true; + if (len == 114 && MATCH(payload, 0x00, 0x70, 0x00, 0x01)) + return true; return false; } diff --git a/lib/udp/Makefile.am b/lib/udp/Makefile.am index c7887374..626253a4 100644 --- a/lib/udp/Makefile.am +++ b/lib/udp/Makefile.am @@ -99,6 +99,7 @@ libprotoident_udp_la_SOURCES = \ lpi_mystery_99.cc \ lpi_mystery_e9.cc \ lpi_mystery_qq.cc \ + lpi_n2ping.cc \ lpi_natpmp.cc \ lpi_netbios.cc \ lpi_netcat_cctv_udp.cc \ @@ -132,6 +133,7 @@ libprotoident_udp_la_SOURCES = \ lpi_quake.cc \ lpi_quic.cc \ lpi_radius.cc \ + lpi_ramseydash.cc \ lpi_real.cc \ lpi_roblox.cc \ lpi_robocraft.cc \ @@ -166,6 +168,7 @@ libprotoident_udp_la_SOURCES = \ lpi_traceroute.cc \ lpi_tremulous.cc \ lpi_tvants.cc \ + lpi_ubisoft_games.cc \ lpi_unreal.cc \ lpi_ventrilo.cc \ lpi_viber.cc \ diff --git a/lib/udp/lpi_n2ping.cc b/lib/udp/lpi_n2ping.cc new file mode 100644 index 00000000..40a38274 --- /dev/null +++ b/lib/udp/lpi_n2ping.cc @@ -0,0 +1,74 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* This is a horrible pseudo-VPN that is used to access content that is + * restricted to China only. Despite their claims, the traffic is not + * encrypted -- just tunnelled over a custom UDP application protocol + * to a server in Hong Kong. + */ + +static inline bool match_n2ping_header(uint32_t payload) { + + if (MATCH(payload, 0x08, 0x00, 0x00, 0x00)) + return true; + return false; +} + +static inline bool match_n2ping(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (data->server_port != 44778 && data->client_port != 44778) { + return false; + } + + if (match_n2ping_header(data->payload[0]) && + match_n2ping_header(data->payload[1])) { + if (data->payload_len[0] < 100) + return true; + if (data->payload_len[1] < 100) + return true; + } + + return false; +} + +static lpi_module_t lpi_n2ping = { + LPI_PROTO_UDP_N2PING, + LPI_CATEGORY_TUNNELLING, + "N2Ping", + 150, + match_n2ping +}; + +void register_n2ping(LPIModuleMap *mod_map) { + register_protocol(&lpi_n2ping, mod_map); +} + diff --git a/lib/udp/lpi_ramseydash.cc b/lib/udp/lpi_ramseydash.cc new file mode 100644 index 00000000..3f5a3400 --- /dev/null +++ b/lib/udp/lpi_ramseydash.cc @@ -0,0 +1,79 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* Gordon Ramsey Dash -- mobile game */ + +static inline bool match_rdash_56da(uint32_t payload, uint32_t len) { + + if (len == 24 && MATCH(payload, 0x56, 0xda, 0x00, 0x00)) + return true; + return false; + +} + +static inline bool match_rdash_da57(uint32_t payload, uint32_t len) { + + if (len >= 150 && len <= 200 && MATCH(payload, 0xda, 0x57, ANY, ANY)) + return true; + return false; + +} + +static inline bool match_ramsey_dash(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_rdash_56da(data->payload[0], data->payload_len[0])) { + if (match_rdash_da57(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_rdash_56da(data->payload[1], data->payload_len[1])) { + if (match_rdash_da57(data->payload[0], data->payload_len[0])) + return true; + } + + + + return false; +} + +static lpi_module_t lpi_ramsey_dash = { + LPI_PROTO_UDP_RAMSEY_DASH, + LPI_CATEGORY_GAMING, + "RamseyDash", + 12, + match_ramsey_dash +}; + +void register_ramsey_dash(LPIModuleMap *mod_map) { + register_protocol(&lpi_ramsey_dash, mod_map); +} + diff --git a/lib/udp/lpi_ubisoft_games.cc b/lib/udp/lpi_ubisoft_games.cc new file mode 100644 index 00000000..d64475d8 --- /dev/null +++ b/lib/udp/lpi_ubisoft_games.cc @@ -0,0 +1,66 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* A variety of Ubisoft games seem to use a similar protocol for + * multiplayer. Usually seen on UDP port 3074, but not always. + */ + +static inline bool match_ubisoft_pat(uint32_t payload, uint32_t len) { + + if (len == 71 && MATCH(payload, 0x00, 0x00, 0x00, 0x00)) + return true; + return false; + +} + +static inline bool match_ubisoft_games(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_ubisoft_pat(data->payload[0], data->payload_len[0])) { + if (match_ubisoft_pat(data->payload[1], data->payload_len[1])) + return true; + } + + return false; +} + +static lpi_module_t lpi_ubisoft_games = { + LPI_PROTO_UDP_UBISOFT_GAMES, + LPI_CATEGORY_GAMING, + "UbisoftGames", + 199, + match_ubisoft_games +}; + +void register_ubisoft_games(LPIModuleMap *mod_map) { + register_protocol(&lpi_ubisoft_games, mod_map); +} + diff --git a/lib/udp/udp_protocols.h b/lib/udp/udp_protocols.h index 218243c3..0d7c3ab1 100644 --- a/lib/udp/udp_protocols.h +++ b/lib/udp/udp_protocols.h @@ -129,6 +129,7 @@ void register_mystery_8000_udp(LPIModuleMap *mod_map); void register_mystery_99(LPIModuleMap *mod_map); void register_mystery_e9(LPIModuleMap *mod_map); void register_mystery_qq(LPIModuleMap *mod_map); +void register_n2ping(LPIModuleMap *mod_map); void register_natpmp(LPIModuleMap *mod_map); void register_netbios_udp(LPIModuleMap *mod_map); void register_netcat_cctv_udp(LPIModuleMap *mod_map); @@ -161,6 +162,7 @@ void register_qqpcmgr(LPIModuleMap *mod_map); void register_quake(LPIModuleMap *mod_map); void register_quic(LPIModuleMap *mod_map); void register_radius(LPIModuleMap *mod_map); +void register_ramsey_dash(LPIModuleMap *mod_map); void register_real(LPIModuleMap *mod_map); void register_roblox(LPIModuleMap *mod_map); void register_robocraft(LPIModuleMap *mod_map); @@ -195,6 +197,7 @@ void register_thq(LPIModuleMap *mod_map); void register_traceroute(LPIModuleMap *mod_map); void register_tremulous(LPIModuleMap *mod_map); void register_tvants(LPIModuleMap *mod_map); +void register_ubisoft_games(LPIModuleMap *mod_map); void register_udp_no_payload(LPIModuleMap *mod_map); void register_unreal(LPIModuleMap *mod_map); void register_ventrilo(LPIModuleMap *mod_map); From 99d0fd0c03d45e175673dc70cdf049519081b861 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Tue, 11 Apr 2017 15:04:48 +1200 Subject: [PATCH 20/68] Added some extra byteswapping functionality to proto_common --- lib/proto_common.cc | 21 +++++++++++++++ lib/proto_common.h | 63 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/lib/proto_common.cc b/lib/proto_common.cc index efd719d1..1b4644a6 100644 --- a/lib/proto_common.cc +++ b/lib/proto_common.cc @@ -807,3 +807,24 @@ bool match_yy_payload(uint32_t payload, uint32_t len) { return false; } + +/* Byte swapping functions for various inttypes */ +uint64_t byteswap64(uint64_t num) +{ + return (byteswap32((num&0xFFFFFFFF00000000ULL)>>32)) + |((uint64_t)byteswap32(num&0x00000000FFFFFFFFULL)<<32); +} + +uint32_t byteswap32(uint32_t num) +{ + return ((num&0x000000FFU)<<24) + | ((num&0x0000FF00U)<<8) + | ((num&0x00FF0000U)>>8) + | ((num&0xFF000000U)>>24); +} + +uint16_t byteswap16(uint16_t num) +{ + return ((num<<8)&0xFF00)|((num>>8)&0x00FF); +} + diff --git a/lib/proto_common.h b/lib/proto_common.h index b48287f4..679f784b 100644 --- a/lib/proto_common.h +++ b/lib/proto_common.h @@ -57,6 +57,69 @@ (memcmp(&(x),(st),sizeof(x))==0) +/** Byteswaps a 64-bit value. + * + * @param num The value to be byteswapped. + * @return The byteswapped 64-bit number + * + */ +uint64_t byteswap64(uint64_t num); + +/** Byteswaps a 32-bit value. + * + * @param num The value to be byteswapped. + * @return The byteswapped 32-bit number + * + */ +uint32_t byteswap32(uint32_t num); + +/** Byteswaps a 16-bit value. + * + * @param num The value to be byteswapped. + * @return The byteswapped 16-bit number + * + */ +uint16_t byteswap16(uint16_t num); + + +#if __BYTE_ORDER == __BIG_ENDIAN +#define bswap_host_to_be64(num) ((uint64_t)(num)) +#define bswap_host_to_le64(num) byteswap64(num) +#define bswap_host_to_be32(num) ((uint32_t)(num)) +#define bswap_host_to_le32(num) byteswap32(num) +#define bswap_host_to_be16(num) ((uint16_t)(num)) +#define bswap_host_to_le16(num) byteswap16(num) + +#define bswap_be_to_host64(num) ((uint64_t)(num)) +#define bswap_le_to_host64(num) byteswap64(num) +#define bswap_be_to_host32(num) ((uint32_t)(num)) +#define bswap_le_to_host32(num) byteswap32(num) +#define bswap_be_to_host16(num) ((uint16_t)(num)) +#define bswap_le_to_host16(num) byteswap16(num) + +/* We use ntoh*() here, because the compiler may + * attempt to optimise it + */ +#elif __BYTE_ORDER == __LITTLE_ENDIAN +#define bswap_host_to_be64(num) (byteswap64(num)) +#define bswap_host_to_le64(num) ((uint64_t)(num)) +#define bswap_host_to_be32(num) (htonl(num)) +#define bswap_host_to_le32(num) ((uint32_t)(num)) +#define bswap_host_to_be16(num) (htons(num)) +#define bswap_host_to_le16(num) ((uint16_t)(num)) + +#define bswap_be_to_host64(num) (byteswap64(num)) +#define bswap_le_to_host64(num) ((uint64_t)(num)) +#define bswap_be_to_host32(num) (ntohl(num)) +#define bswap_le_to_host32(num) ((uint32_t)(num)) +#define bswap_be_to_host16(num) (ntohs(num)) +#define bswap_le_to_host16(num) ((uint16_t)(num)) + +#else +#error "Unknown byte order" +#endif + + bool match_str_either(lpi_data_t *data, const char *string); bool match_str_both(lpi_data_t *data, const char *string1, const char *string2); From 0c6a0219c708f7d2e30eaef479289e74650ad548 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Tue, 11 Apr 2017 15:05:26 +1200 Subject: [PATCH 21/68] Another batch of new protocols Added: * The Crew * TurboVPN * Gears of War Improved: * Minecraft * Zoom Fixed: * Byte ordering bug with LifeForge --- lib/libprotoident.h | 3 ++ lib/proto_manager.cc | 3 ++ lib/tcp/lpi_lifeforge.cc | 2 +- lib/tcp/lpi_minecraft.cc | 5 +++ lib/udp/Makefile.am | 3 ++ lib/udp/lpi_gearsofwar.cc | 88 +++++++++++++++++++++++++++++++++++++++ lib/udp/lpi_thecrew.cc | 71 +++++++++++++++++++++++++++++++ lib/udp/lpi_turbovpn.cc | 66 +++++++++++++++++++++++++++++ lib/udp/lpi_zoom.cc | 34 ++++++++++++++- lib/udp/udp_protocols.h | 3 ++ 10 files changed, 276 insertions(+), 2 deletions(-) create mode 100644 lib/udp/lpi_gearsofwar.cc create mode 100644 lib/udp/lpi_thecrew.cc create mode 100644 lib/udp/lpi_turbovpn.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 86381aff..0628ac7e 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -505,6 +505,9 @@ typedef enum { LPI_PROTO_UDP_N2PING, LPI_PROTO_UDP_RAMSEY_DASH, LPI_PROTO_UDP_UBISOFT_GAMES, + LPI_PROTO_UDP_THE_CREW, + LPI_PROTO_UDP_TURBOVPN, + LPI_PROTO_UDP_GEARSOFWAR, /* Patterns that we can match, but do not know the protocol */ LPI_PROTO_REJECTION, /* All responses are 0x02 */ diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index 02ae4d96..78339fa2 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -338,6 +338,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_funshion_udp(mod_map); register_gamespy(mod_map); register_garena(mod_map); + register_gearsofwar(mod_map); register_gnutella_udp(mod_map); register_gnutella2_udp(mod_map); register_gnutella_weak(mod_map); @@ -445,9 +446,11 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_teamviewer_udp(mod_map); register_teredo(mod_map); register_tftp(mod_map); + register_thecrew(mod_map); register_thq(mod_map); register_traceroute(mod_map); register_tremulous(mod_map); + register_turbovpn(mod_map); register_tvants(mod_map); register_ubisoft_games(mod_map); register_udp_no_payload(mod_map); diff --git a/lib/tcp/lpi_lifeforge.cc b/lib/tcp/lpi_lifeforge.cc index be2eb563..a216e83f 100644 --- a/lib/tcp/lpi_lifeforge.cc +++ b/lib/tcp/lpi_lifeforge.cc @@ -32,7 +32,7 @@ static inline bool match_lifeforge_login(uint32_t payload, uint32_t len) { - uint32_t plen = ntohl(payload); + uint32_t plen = bswap_le_to_host32(payload); /* I've only seen 0x25 in here, but that may vary depending on * username length? */ diff --git a/lib/tcp/lpi_minecraft.cc b/lib/tcp/lpi_minecraft.cc index 30e36be6..aae463ee 100644 --- a/lib/tcp/lpi_minecraft.cc +++ b/lib/tcp/lpi_minecraft.cc @@ -100,6 +100,11 @@ static inline bool match_mc_handshake_reply(uint32_t payload, uint32_t len) { return true; } + if (len == 4) { + if (MATCH(payload, 0x03, 0x03, 0x80, 0x02)) + return true; + } + return false; } diff --git a/lib/udp/Makefile.am b/lib/udp/Makefile.am index 626253a4..5cf0d522 100644 --- a/lib/udp/Makefile.am +++ b/lib/udp/Makefile.am @@ -56,6 +56,7 @@ libprotoident_udp_la_SOURCES = \ lpi_funshion.cc \ lpi_gamespy.cc \ lpi_garena.cc \ + lpi_gearsofwar.cc \ lpi_gnutella.cc \ lpi_gnutella2.cc \ lpi_gnutella_weak.cc \ @@ -164,9 +165,11 @@ libprotoident_udp_la_SOURCES = \ lpi_teamviewer.cc \ lpi_teredo.cc \ lpi_tftp.cc \ + lpi_thecrew.cc \ lpi_thq.cc \ lpi_traceroute.cc \ lpi_tremulous.cc \ + lpi_turbovpn.cc \ lpi_tvants.cc \ lpi_ubisoft_games.cc \ lpi_unreal.cc \ diff --git a/lib/udp/lpi_gearsofwar.cc b/lib/udp/lpi_gearsofwar.cc new file mode 100644 index 00000000..5c3107f9 --- /dev/null +++ b/lib/udp/lpi_gearsofwar.cc @@ -0,0 +1,88 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_gow_44(uint32_t payload, uint32_t len) { + if (MATCH(payload, 0x00, 0x00, 0x00, 0x00) && len == 44) + return true; + return false; +} + +static inline bool match_gow_51(uint32_t payload, uint32_t len) { + if (MATCH(payload, 0x00, 0x00, 0x00, 0x00) && len == 51) + return true; + return false; +} + +static inline bool match_gow_port(uint32_t sp, uint32_t cp) { + + if (sp == 30200 || cp == 30200) + return true; + if (sp == 30400 || cp == 30400) + return true; + if (sp == 30600 || cp == 30600) + return true; + if (sp == 30000 || cp == 30000) + return true; + + return false; +} + +static inline bool match_gearsofwar(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (!match_gow_port(data->server_port, data->client_port)) + return false; + + if (match_gow_44(data->payload[0], data->payload_len[0])) { + if (match_gow_51(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_gow_44(data->payload[1], data->payload_len[1])) { + if (match_gow_51(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_gearsofwar = { + LPI_PROTO_UDP_GEARSOFWAR, + LPI_CATEGORY_GAMING, + "GearsOfWar", + 199, + match_gearsofwar +}; + +void register_gearsofwar(LPIModuleMap *mod_map) { + register_protocol(&lpi_gearsofwar, mod_map); +} + diff --git a/lib/udp/lpi_thecrew.cc b/lib/udp/lpi_thecrew.cc new file mode 100644 index 00000000..d338c8ce --- /dev/null +++ b/lib/udp/lpi_thecrew.cc @@ -0,0 +1,71 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_thecrew_hello(uint32_t payload, uint32_t len) { + + if (MATCHSTR(payload, "\xff\xff\xff\xff")) { + if (len == 50) + return true; + if (len == 39) + return true; + if (len == 60) + return true; + } + return false; + +} + +static inline bool match_thecrew(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (data->server_port != 3001 && data->client_port != 3001) + return false; + + if (match_thecrew_hello(data->payload[0], data->payload_len[0])) { + if (match_thecrew_hello(data->payload[1], data->payload_len[1])) + return true; + } + + return false; +} + +static lpi_module_t lpi_thecrew = { + LPI_PROTO_UDP_THE_CREW, + LPI_CATEGORY_GAMING, + "TheCrew", + 75, + match_thecrew +}; + +void register_thecrew(LPIModuleMap *mod_map) { + register_protocol(&lpi_thecrew, mod_map); +} + diff --git a/lib/udp/lpi_turbovpn.cc b/lib/udp/lpi_turbovpn.cc new file mode 100644 index 00000000..2ed502a9 --- /dev/null +++ b/lib/udp/lpi_turbovpn.cc @@ -0,0 +1,66 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_gene(uint32_t payload, uint32_t len) { + + /* Range of packet sizes, but always fairly small */ + if (MATCH(payload, 'G', 'E', 'N', 'E')) { + if (len >= 15 && len <= 50) + return true; + } + + return false; + +} + +static inline bool match_turbovpn(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_gene(data->payload[0], data->payload_len[0])) { + if (match_gene(data->payload[1], data->payload_len[1])) + return true; + } + + return false; +} + +static lpi_module_t lpi_turbovpn = { + LPI_PROTO_UDP_TURBOVPN, + LPI_CATEGORY_TUNNELLING, + "TurboVPN", + 10, + match_turbovpn +}; + +void register_turbovpn(LPIModuleMap *mod_map) { + register_protocol(&lpi_turbovpn, mod_map); +} + diff --git a/lib/udp/lpi_zoom.cc b/lib/udp/lpi_zoom.cc index 11efedb6..d08ae680 100644 --- a/lib/udp/lpi_zoom.cc +++ b/lib/udp/lpi_zoom.cc @@ -61,7 +61,17 @@ static inline bool match_zoom_05(uint32_t payload) { } -static inline bool match_zoom(lpi_data_t *data, lpi_module_t *mod UNUSED) { +static inline bool match_zoom_stream_hello(uint32_t payload, uint32_t len) { + + /* Seen 85 and 86 so far */ + if (len >= 80 && len <= 90) { + if (MATCH(payload, 0x1f, 0x01, 0x01, 0x00)) + return true; + } + +} + +static inline bool match_zoom_manager(lpi_data_t *data) { if (data->server_port != 8801 && data->client_port != 8801) { return false; @@ -87,6 +97,28 @@ static inline bool match_zoom(lpi_data_t *data, lpi_module_t *mod UNUSED) { return false; } +static inline bool match_zoom_stream(lpi_data_t *data) { + + if (match_zoom_stream_hello(data->payload[0], data->payload_len[0])) { + if (match_zoom_stream_hello(data->payload[1], data->payload_len[1])) + return true; + } + + return false; + +} + +static inline bool match_zoom(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_zoom_manager(data)) + return true; + + if (match_zoom_stream(data)) + return true; + + return false; +} + static lpi_module_t lpi_zoom = { LPI_PROTO_UDP_ZOOM, LPI_CATEGORY_VOIP, diff --git a/lib/udp/udp_protocols.h b/lib/udp/udp_protocols.h index 0d7c3ab1..463b426f 100644 --- a/lib/udp/udp_protocols.h +++ b/lib/udp/udp_protocols.h @@ -86,6 +86,7 @@ void register_freechal(LPIModuleMap *mod_map); void register_funshion_udp(LPIModuleMap *mod_map); void register_gamespy(LPIModuleMap *mod_map); void register_garena(LPIModuleMap *mod_map); +void register_gearsofwar(LPIModuleMap *mod_map); void register_gnutella_udp(LPIModuleMap *mod_map); void register_gnutella2_udp(LPIModuleMap *mod_map); void register_gnutella_weak(LPIModuleMap *mod_map); @@ -193,9 +194,11 @@ void register_teamspeak(LPIModuleMap *mod_map); void register_teamviewer_udp(LPIModuleMap *mod_map); void register_teredo(LPIModuleMap *mod_map); void register_tftp(LPIModuleMap *mod_map); +void register_thecrew(LPIModuleMap *mod_map); void register_thq(LPIModuleMap *mod_map); void register_traceroute(LPIModuleMap *mod_map); void register_tremulous(LPIModuleMap *mod_map); +void register_turbovpn(LPIModuleMap *mod_map); void register_tvants(LPIModuleMap *mod_map); void register_ubisoft_games(LPIModuleMap *mod_map); void register_udp_no_payload(LPIModuleMap *mod_map); From 9e7d69a8b87560f14ebf02078ac483154b24cf98 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Tue, 18 Apr 2017 17:28:38 +1200 Subject: [PATCH 22/68] Allow RTMP packets < 4 bytes in size Managed to observe a single byte RTMP packet, so we now have to allow for RTMP handshakes being split over multiple packets. In this case, I am enforcing the use of the default RTMP port to reduce the possibility of false positives. --- lib/tcp/lpi_rtmp.cc | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/tcp/lpi_rtmp.cc b/lib/tcp/lpi_rtmp.cc index 5c46c8d3..147a0f6d 100644 --- a/lib/tcp/lpi_rtmp.cc +++ b/lib/tcp/lpi_rtmp.cc @@ -30,9 +30,10 @@ #include "proto_manager.h" #include "proto_common.h" -static inline bool match_rtmp_server_handshake(uint32_t payload, uint32_t len) { +static inline bool match_rtmp_server_handshake(uint32_t payload, uint32_t len, + bool defaultport) { - if (len < 4) + if (len < 4 && !defaultport) return false; /* Standard RTMP handshake types */ @@ -73,10 +74,15 @@ static inline bool match_rtmp_client_handshake(uint32_t payload, uint32_t len) { static inline bool match_rtmp(lpi_data_t *data, lpi_module_t *mod UNUSED) { + bool defaultport = false; + + if (data->server_port == 1935 || data->client_port == 1935) + defaultport = true; + if (match_rtmp_client_handshake(data->payload[0], data->payload_len[0])) { if (match_rtmp_server_handshake(data->payload[1], - data->payload_len[1])) { + data->payload_len[1], defaultport)) { return true; } } @@ -84,7 +90,7 @@ static inline bool match_rtmp(lpi_data_t *data, lpi_module_t *mod UNUSED) { if (match_rtmp_client_handshake(data->payload[1], data->payload_len[1])) { if (match_rtmp_server_handshake(data->payload[0], - data->payload_len[0])) { + data->payload_len[0], defaultport)) { return true; } } From 8eebad885f64da9a6b84576e5fae584172cdbe8a Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Tue, 18 Apr 2017 17:30:53 +1200 Subject: [PATCH 23/68] Add new protocol: Maplestory China --- lib/libprotoident.h | 1 + lib/proto_manager.cc | 1 + lib/tcp/Makefile.am | 1 + lib/tcp/lpi_maplestory_china.cc | 71 +++++++++++++++++++++++++++++++++ lib/tcp/tcp_protocols.h | 1 + 5 files changed, 75 insertions(+) create mode 100644 lib/tcp/lpi_maplestory_china.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 0628ac7e..ca6afd1b 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -325,6 +325,7 @@ typedef enum { LPI_PROTO_BITCOIN, LPI_PROTO_LIFEFORGE, LPI_PROTO_ACESTREAM, + LPI_PROTO_MAPLESTORY_CHINA, /* UDP Protocols */ LPI_PROTO_UDP, diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index 78339fa2..17dd5cfc 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -152,6 +152,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_lifeforge(mod_map); register_line(mod_map); register_llp2p(mod_map); + register_maplestory_china(mod_map); register_maxicloud(mod_map); register_message4u(mod_map); register_minecraft(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index 5d4cd60c..ee8d582d 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -85,6 +85,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_lifeforge.cc \ lpi_line.cc \ lpi_llp2p.cc \ + lpi_maplestory_china.cc \ lpi_maxicloud.cc \ lpi_message4u.cc \ lpi_minecraft.cc \ diff --git a/lib/tcp/lpi_maplestory_china.cc b/lib/tcp/lpi_maplestory_china.cc new file mode 100644 index 00000000..b5fbf27d --- /dev/null +++ b/lib/tcp/lpi_maplestory_china.cc @@ -0,0 +1,71 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_cms_hello(uint32_t payload, uint32_t len) { + + if (len == 16 || len == 536) { + if (MATCH(payload, 0x0e, 0x00, 0x8d, 0x00)) + return true; + } + return false; + +} + +static inline bool match_maplestory_china(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* Can also restrict to ports 8585 and 8586 if required */ + + if (match_cms_hello(data->payload[0], data->payload_len[0])) { + if (data->payload_len[1] == 42) + return true; + } + + if (match_cms_hello(data->payload[1], data->payload_len[1])) { + if (data->payload_len[0] == 42) + return true; + } + + return false; +} + +static lpi_module_t lpi_maplestory_china = { + LPI_PROTO_MAPLESTORY_CHINA, + LPI_CATEGORY_GAMING, + "MaplestoryChina", + 12, + match_maplestory_china +}; + +void register_maplestory_china(LPIModuleMap *mod_map) { + register_protocol(&lpi_maplestory_china, mod_map); +} + diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index 8c1cfd54..19c7a7cd 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -113,6 +113,7 @@ void register_ldap(LPIModuleMap *mod_map); void register_lifeforge(LPIModuleMap *mod_map); void register_line(LPIModuleMap *mod_map); void register_llp2p(LPIModuleMap *mod_map); +void register_maplestory_china(LPIModuleMap *mod_map); void register_maxicloud(LPIModuleMap *mod_map); void register_message4u(LPIModuleMap *mod_map); void register_minecraft(LPIModuleMap *mod_map); From a31e1b4c76ef00fc4541ae83acc46ed794c5336b Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Tue, 18 Apr 2017 17:31:14 +1200 Subject: [PATCH 24/68] Add file header for TIFF files --- lib/proto_common.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/proto_common.cc b/lib/proto_common.cc index 1b4644a6..6ee7ac46 100644 --- a/lib/proto_common.cc +++ b/lib/proto_common.cc @@ -305,6 +305,10 @@ bool match_file_header(uint32_t payload) { if (MATCH(payload, 0x00, 0x00, 0x00, 0x20)) return true; + /* TIFF */ + if (MATCH(payload, 0x49, 0x49, 0x2a, 0x00)) + return true; + /* I'm pretty sure the following are files of some type or another. * They crop up pretty often in our test data sets, so I'm going to * put them in here. From 6fba58cb93236f4d0b22b95a19dfa5e424b6898a Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 21 Apr 2017 18:04:29 +1200 Subject: [PATCH 25/68] Fix missing 'return false' in lpi_zoom.cc --- lib/udp/lpi_zoom.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/udp/lpi_zoom.cc b/lib/udp/lpi_zoom.cc index d08ae680..f4802339 100644 --- a/lib/udp/lpi_zoom.cc +++ b/lib/udp/lpi_zoom.cc @@ -68,6 +68,7 @@ static inline bool match_zoom_stream_hello(uint32_t payload, uint32_t len) { if (MATCH(payload, 0x1f, 0x01, 0x01, 0x00)) return true; } + return false; } From 26b0b7bb16458bb004786f2e7c2939d4072eaac8 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 21 Apr 2017 18:05:39 +1200 Subject: [PATCH 26/68] New batch of protocols Added: * Douyu * NDT Throughput Test * Relay Improved: * Kakao * Mongo (fixed bad byte ordering) --- lib/libprotoident.h | 3 ++ lib/proto_manager.cc | 3 ++ lib/tcp/Makefile.am | 3 ++ lib/tcp/lpi_douyu.cc | 109 ++++++++++++++++++++++++++++++++++++++++ lib/tcp/lpi_kakao.cc | 2 +- lib/tcp/lpi_mongo.cc | 4 +- lib/tcp/lpi_ndt_tput.cc | 60 ++++++++++++++++++++++ lib/tcp/lpi_relay.cc | 64 +++++++++++++++++++++++ lib/tcp/tcp_protocols.h | 3 ++ 9 files changed, 248 insertions(+), 3 deletions(-) create mode 100644 lib/tcp/lpi_douyu.cc create mode 100644 lib/tcp/lpi_ndt_tput.cc create mode 100644 lib/tcp/lpi_relay.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index ca6afd1b..4cd21437 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -326,6 +326,9 @@ typedef enum { LPI_PROTO_LIFEFORGE, LPI_PROTO_ACESTREAM, LPI_PROTO_MAPLESTORY_CHINA, + LPI_PROTO_NDT_TPUT, + LPI_PROTO_RELAY, + LPI_PROTO_DOUYU, /* UDP Protocols */ LPI_PROTO_UDP, diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index 17dd5cfc..ad56e8fa 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -98,6 +98,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_dianping_tcp(mod_map); register_directconnect(mod_map); register_dns_tcp(mod_map); + register_douyu(mod_map); register_duelingnetwork(mod_map); register_dvrns(mod_map); register_dxp(mod_map); @@ -175,6 +176,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_mystery_symantec(mod_map); register_mzinga(mod_map); register_ncsoft(mod_map); + register_ndt_tput(mod_map); register_netbios(mod_map); register_netcat_cctv(mod_map); register_nntp(mod_map); @@ -203,6 +205,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_rbls(mod_map); register_rdp(mod_map); register_rejection(mod_map); + register_relay(mod_map); register_revolver_nblbt(mod_map); register_rfb(mod_map); register_rpcscan(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index ee8d582d..77231efa 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -31,6 +31,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_dianping_tcp.cc \ lpi_directconnect.cc \ lpi_dns.cc \ + lpi_douyu.cc \ lpi_duelingnetwork.cc \ lpi_dvrns.cc \ lpi_dxp.cc \ @@ -108,6 +109,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_mystery_symantec.cc \ lpi_mzinga.cc \ lpi_ncsoft.cc \ + lpi_ndt_tput.cc \ lpi_netbios.cc \ lpi_netcat_cctv.cc \ lpi_nntp.cc \ @@ -136,6 +138,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_rbls.cc \ lpi_rdp.cc \ lpi_rejection.cc \ + lpi_relay.cc \ lpi_revolver_nblbt.cc \ lpi_rfb.cc \ lpi_rpcscan.cc \ diff --git a/lib/tcp/lpi_douyu.cc b/lib/tcp/lpi_douyu.cc new file mode 100644 index 00000000..af65a7b0 --- /dev/null +++ b/lib/tcp/lpi_douyu.cc @@ -0,0 +1,109 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* This is a classic 4-byte length protocol, but there is plenty of + * scope for the packet sizes to vary a bit so we can't just look for + * a specific combo of packet sizes */ + +static inline bool match_douyu_req(uint32_t payload, uint32_t len) { + + uint32_t plen = ntohl(payload); + + /* Packet usually contains a username and a password so + * can probably vary quite a bit in size */ + if (plen == len - 4) { + if (len < 128) + return true; + } + + return false; +} + +static inline bool match_douyu_reply(uint32_t payload, uint32_t len) { + + uint32_t plen = ntohl(payload); + + /* Response packets seem like they will vary a lot less in + * size -- could be wrong though */ + + if (plen == len - 4) { + if (len >= 225 && len <= 255) + return true; + } + + return false; +} + +static inline bool match_douyu_port(uint16_t port) { + + /* Based purely on observed flows, rather than any docs */ + if (port >= 8601 && port <= 8605) + return true; + if (port >= 12601 && port <= 12605) + return true; + return false; +} + +static inline bool match_douyu(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* Tends to use a couple of different port ranges */ + if (!match_douyu_port(data->server_port) && + !match_douyu_port(data->client_port)) { + return false; + } + + if (match_douyu_req(data->payload[0], data->payload_len[0])) { + if (match_douyu_reply(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_douyu_req(data->payload[1], data->payload_len[1])) { + if (match_douyu_reply(data->payload[0], data->payload_len[0])) + return true; + } + + + return false; +} + +static lpi_module_t lpi_douyu = { + LPI_PROTO_DOUYU, + LPI_CATEGORY_STREAMING, + "Douyu", + 249, + match_douyu +}; + +void register_douyu(LPIModuleMap *mod_map) { + register_protocol(&lpi_douyu, mod_map); +} + diff --git a/lib/tcp/lpi_kakao.cc b/lib/tcp/lpi_kakao.cc index d6bec336..6ec17c81 100644 --- a/lib/tcp/lpi_kakao.cc +++ b/lib/tcp/lpi_kakao.cc @@ -32,7 +32,7 @@ static inline bool match_kakao_req(uint32_t payload, uint32_t len) { - if (len < 399) + if (len < 250) return false; if (MATCH(payload, 0x00, 0x01, 0x00, 0x00)) return true; diff --git a/lib/tcp/lpi_mongo.cc b/lib/tcp/lpi_mongo.cc index 3b3456ce..e48405d0 100644 --- a/lib/tcp/lpi_mongo.cc +++ b/lib/tcp/lpi_mongo.cc @@ -33,7 +33,7 @@ /* Mongo protocol uses the first four bytes as a length field */ static inline bool match_mongo_req(uint32_t payload, uint32_t len) { - uint32_t mongolen = ntohl(payload); + uint32_t mongolen = bswap_le_to_host32(payload); /* Most requests are very small */ if (MATCH(payload, ANY, 0x00, 0x00, 0x00) || @@ -52,7 +52,7 @@ static inline bool match_mongo_req(uint32_t payload, uint32_t len) { static inline bool match_mongo_reply(uint32_t payload, uint32_t len) { - uint32_t mongolen = ntohl(payload); + uint32_t mongolen = bswap_le_to_host32(payload); /* If reply is short, mongolen should match the packet length */ if (mongolen == len) diff --git a/lib/tcp/lpi_ndt_tput.cc b/lib/tcp/lpi_ndt_tput.cc new file mode 100644 index 00000000..640572c1 --- /dev/null +++ b/lib/tcp/lpi_ndt_tput.cc @@ -0,0 +1,60 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + + +static inline bool match_ndt_tput(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* One-way throughput test */ + if (MATCH(data->payload[0], 0x20, 0x21, 0x22, 0x23) && + data->payload_len[1] == 0) { + return true; + } + if (MATCH(data->payload[1], 0x20, 0x21, 0x22, 0x23) && + data->payload_len[0] == 0) { + return true; + } + + return false; +} + +static lpi_module_t lpi_ndt_tput = { + LPI_PROTO_NDT_TPUT, + LPI_CATEGORY_MONITORING, + "NDTThroughput", + 100, + match_ndt_tput +}; + +void register_ndt_tput(LPIModuleMap *mod_map) { + register_protocol(&lpi_ndt_tput, mod_map); +} + diff --git a/lib/tcp/lpi_relay.cc b/lib/tcp/lpi_relay.cc new file mode 100644 index 00000000..41d2c40a --- /dev/null +++ b/lib/tcp/lpi_relay.cc @@ -0,0 +1,64 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* https://docs.syncthing.net/specs/relay-v1.html */ + +static inline bool match_relay_magic(uint32_t payload) { + if (MATCH(payload, 0x9e, 0x79, 0xbc, 0x40)) + return true; + return false; + +} + +static inline bool match_relay(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_relay_magic(data->payload[0])) { + if (match_relay_magic(data->payload[1])) + return true; + } + + + return false; +} + +static lpi_module_t lpi_relay = { + LPI_PROTO_RELAY, + LPI_CATEGORY_NAT, + "Relay", + 5, + match_relay +}; + +void register_relay(LPIModuleMap *mod_map) { + register_protocol(&lpi_relay, mod_map); +} + diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index 19c7a7cd..c3323f2f 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -59,6 +59,7 @@ void register_diablo3(LPIModuleMap *mod_map); void register_dianping_tcp(LPIModuleMap *mod_map); void register_directconnect(LPIModuleMap *mod_map); void register_dns_tcp(LPIModuleMap *mod_map); +void register_douyu(LPIModuleMap *mod_map); void register_duelingnetwork(LPIModuleMap *mod_map); void register_dvrns(LPIModuleMap *mod_map); void register_dxp(LPIModuleMap *mod_map); @@ -136,6 +137,7 @@ void register_mystery_rxxf(LPIModuleMap *mod_map); void register_mystery_symantec(LPIModuleMap *mod_map); void register_mzinga(LPIModuleMap *mod_map); void register_ncsoft(LPIModuleMap *mod_map); +void register_ndt_tput(LPIModuleMap *mod_map); void register_netbios(LPIModuleMap *mod_map); void register_netcat_cctv(LPIModuleMap *mod_map); void register_nntp(LPIModuleMap *mod_map); @@ -164,6 +166,7 @@ void register_razor(LPIModuleMap *mod_map); void register_rbls(LPIModuleMap *mod_map); void register_rdp(LPIModuleMap *mod_map); void register_rejection(LPIModuleMap *mod_map); +void register_relay(LPIModuleMap *mod_map); void register_revolver_nblbt(LPIModuleMap *mod_map); void register_rfb(LPIModuleMap *mod_map); void register_rpcscan(LPIModuleMap *mod_map); From a84075961d76799a4a418c8647472cfcfd31ff14 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 28 Apr 2017 16:10:58 +1200 Subject: [PATCH 27/68] Add a couple more port numbers for Gears of War --- lib/udp/lpi_gearsofwar.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/udp/lpi_gearsofwar.cc b/lib/udp/lpi_gearsofwar.cc index 5c3107f9..b9169e45 100644 --- a/lib/udp/lpi_gearsofwar.cc +++ b/lib/udp/lpi_gearsofwar.cc @@ -50,6 +50,10 @@ static inline bool match_gow_port(uint32_t sp, uint32_t cp) { return true; if (sp == 30600 || cp == 30600) return true; + if (sp == 31000 || cp == 31000) + return true; + if (sp == 30800 || cp == 30800) + return true; if (sp == 30000 || cp == 30000) return true; From 8253aacbd27d8d8cdf133d87813a8204a55fa798 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 28 Apr 2017 16:11:55 +1200 Subject: [PATCH 28/68] Add more packet sizes for Runescape and 360 P2P --- lib/tcp/lpi_runescape.cc | 2 ++ lib/udp/lpi_360p2p.cc | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/lib/tcp/lpi_runescape.cc b/lib/tcp/lpi_runescape.cc index 92a75f2a..5ee5ad3c 100644 --- a/lib/tcp/lpi_runescape.cc +++ b/lib/tcp/lpi_runescape.cc @@ -75,6 +75,8 @@ static inline bool match_runescape_resp(uint32_t payload, uint32_t len) { if (MATCH(payload, 0x0f, 0x00, 0x00, 0x00)) { if (len == 1) return true; + if (len == 5) + return true; } return false; diff --git a/lib/udp/lpi_360p2p.cc b/lib/udp/lpi_360p2p.cc index 4233fc28..99add484 100644 --- a/lib/udp/lpi_360p2p.cc +++ b/lib/udp/lpi_360p2p.cc @@ -39,6 +39,10 @@ static inline bool match_360p2p_request(uint32_t payload, uint32_t len) { static inline bool match_360p2p_reply(uint32_t payload, uint32_t len) { if (len == 40 && MATCH(payload, 0x00, 0x00, 0x00, 0x00)) return true; + if (len == 50 && MATCH(payload, 0x00, 0x00, 0x00, 0x00)) + return true; + if (len == 72 && MATCH(payload, 0x00, 0x00, 0x00, 0x00)) + return true; return false; } From 1a267946047607a53aeef75a6320a51ddf42ae0b Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 28 Apr 2017 16:12:11 +1200 Subject: [PATCH 29/68] Add 2 new file headers (VBSP and LZMA) --- lib/proto_common.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/proto_common.cc b/lib/proto_common.cc index 6ee7ac46..27e8638c 100644 --- a/lib/proto_common.cc +++ b/lib/proto_common.cc @@ -309,6 +309,14 @@ bool match_file_header(uint32_t payload) { if (MATCH(payload, 0x49, 0x49, 0x2a, 0x00)) return true; + /* LZMA */ + if (MATCH(payload, 0x5d, 0x00, 0x00, 0x80)) + return true; + + /* Source engine BSP file */ + if (MATCH(payload, 'V', 'B', 'S', 'P')) + return true; + /* I'm pretty sure the following are files of some type or another. * They crop up pretty often in our test data sets, so I'm going to * put them in here. From 4252e4a9b9189d2a46bbf7007a8f69b18fa3ff11 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Wed, 3 May 2017 12:03:28 +1200 Subject: [PATCH 30/68] Yet another protocol update New protocols: IDriveSync RDP over UDP Twitcasting Improved Protocols: MapleStory China PPStream BaiduYun P2P RTMP (specifically Facebook live streaming) --- lib/libprotoident.h | 3 ++ lib/proto_manager.cc | 3 ++ lib/tcp/Makefile.am | 2 + lib/tcp/lpi_idrivesync.cc | 69 ++++++++++++++++++++++++++++ lib/tcp/lpi_maplestory_china.cc | 2 + lib/tcp/lpi_rtmp.cc | 7 ++- lib/tcp/lpi_twitcasting.cc | 81 +++++++++++++++++++++++++++++++++ lib/tcp/tcp_protocols.h | 2 + lib/udp/Makefile.am | 1 + lib/udp/lpi_baiduyun_p2p.cc | 4 +- lib/udp/lpi_ppstream.cc | 10 ++++ lib/udp/lpi_rdp_udp.cc | 65 ++++++++++++++++++++++++++ lib/udp/udp_protocols.h | 1 + 13 files changed, 248 insertions(+), 2 deletions(-) create mode 100644 lib/tcp/lpi_idrivesync.cc create mode 100644 lib/tcp/lpi_twitcasting.cc create mode 100644 lib/udp/lpi_rdp_udp.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 4cd21437..74d1bd15 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -329,6 +329,8 @@ typedef enum { LPI_PROTO_NDT_TPUT, LPI_PROTO_RELAY, LPI_PROTO_DOUYU, + LPI_PROTO_IDRIVE_SYNC, + LPI_PROTO_TWITCASTING, /* UDP Protocols */ LPI_PROTO_UDP, @@ -512,6 +514,7 @@ typedef enum { LPI_PROTO_UDP_THE_CREW, LPI_PROTO_UDP_TURBOVPN, LPI_PROTO_UDP_GEARSOFWAR, + LPI_PROTO_UDP_RDP, /* Patterns that we can match, but do not know the protocol */ LPI_PROTO_REJECTION, /* All responses are 0x02 */ diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index ad56e8fa..2fcdf7c5 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -131,6 +131,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_http_tunnel(mod_map); register_ica(mod_map); register_id(mod_map); + register_idrivesync(mod_map); register_imap(mod_map); register_imaps(mod_map); register_imesh(mod_map); @@ -251,6 +252,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_trion(mod_map); register_trojan_win32_generic_sb(mod_map); register_trojan_zeroaccess(mod_map); + register_twitcasting(mod_map); register_twitch_irc(mod_map); register_viber(mod_map); register_vodlocker(mod_map); @@ -420,6 +422,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_quic(mod_map); register_radius(mod_map); register_ramsey_dash(mod_map); + register_rdp_udp(mod_map); register_real(mod_map); register_roblox(mod_map); register_robocraft(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index 77231efa..2c8b9ae0 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -64,6 +64,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_http_tunnel.cc \ lpi_ica.cc \ lpi_id.cc \ + lpi_idrivesync.cc \ lpi_imap.cc \ lpi_imaps.cc \ lpi_imesh.cc \ @@ -184,6 +185,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_trion.cc \ lpi_trojan_win32_generic_sb.cc \ lpi_trojan_zeroaccess.cc \ + lpi_twitcasting.cc \ lpi_twitch_irc.cc \ lpi_viber.cc \ lpi_vodlocker.cc \ diff --git a/lib/tcp/lpi_idrivesync.cc b/lib/tcp/lpi_idrivesync.cc new file mode 100644 index 00000000..5a4379bb --- /dev/null +++ b/lib/tcp/lpi_idrivesync.cc @@ -0,0 +1,69 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_idrivesync_hello(uint32_t payload) { + + if (MATCH(payload, '@', 'I', 'D', 'E')) + return true; + return false; + +} + +static inline bool match_idrivesync(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_idrivesync_hello(data->payload[0])) { + if (match_idrivesync_hello(data->payload[1])) + return true; + if (data->payload_len[1] == 0) + return true; + } + + if (match_idrivesync_hello(data->payload[1])) { + if (data->payload_len[0] == 0) + return true; + } + + return false; +} + +static lpi_module_t lpi_idrivesync = { + LPI_PROTO_IDRIVE_SYNC, + LPI_CATEGORY_CLOUD, + "IDriveSync", + 5, + match_idrivesync +}; + +void register_idrivesync(LPIModuleMap *mod_map) { + register_protocol(&lpi_idrivesync, mod_map); +} + diff --git a/lib/tcp/lpi_maplestory_china.cc b/lib/tcp/lpi_maplestory_china.cc index b5fbf27d..07f0dc3a 100644 --- a/lib/tcp/lpi_maplestory_china.cc +++ b/lib/tcp/lpi_maplestory_china.cc @@ -35,6 +35,8 @@ static inline bool match_cms_hello(uint32_t payload, uint32_t len) { if (len == 16 || len == 536) { if (MATCH(payload, 0x0e, 0x00, 0x8d, 0x00)) return true; + if (MATCH(payload, 0x0e, 0x00, 0x8e, 0x00)) + return true; } return false; diff --git a/lib/tcp/lpi_rtmp.cc b/lib/tcp/lpi_rtmp.cc index 147a0f6d..cdf9991a 100644 --- a/lib/tcp/lpi_rtmp.cc +++ b/lib/tcp/lpi_rtmp.cc @@ -79,6 +79,11 @@ static inline bool match_rtmp(lpi_data_t *data, lpi_module_t *mod UNUSED) { if (data->server_port == 1935 || data->client_port == 1935) defaultport = true; + /* Facebook live streaming wants to use port 80 sometimes and they + * love sending 1 byte handshakes :( */ + if (data->server_port == 80 || data->client_port == 80) + defaultport = true; + if (match_rtmp_client_handshake(data->payload[0], data->payload_len[0])) { if (match_rtmp_server_handshake(data->payload[1], @@ -101,7 +106,7 @@ static lpi_module_t lpi_rtmp = { LPI_PROTO_RTMP, LPI_CATEGORY_STREAMING, "RTMP", - 16, /* Not a strong rule */ + 169, /* Not a strong rule */ match_rtmp }; diff --git a/lib/tcp/lpi_twitcasting.cc b/lib/tcp/lpi_twitcasting.cc new file mode 100644 index 00000000..3769bed0 --- /dev/null +++ b/lib/tcp/lpi_twitcasting.cc @@ -0,0 +1,81 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* Live self-streaming protocol, popular in Japan */ + +static inline bool match_tc_get(uint32_t payload) { + /* Yes, they have managed to co-opt "GET" for this protocol */ + + if (MATCH(payload, 'G', 'E', 'T', 0x20)) + return true; + return false; +} + +static inline bool match_tc_reply(uint32_t payload, uint32_t len) { + + /* Possible that bytes 3 and 4 are a length field? */ + + if (len == 19 && MATCH(payload, 'T', 'C', 0x0c, 0x00)) + return true; + return false; + +} + +static inline bool match_twitcasting(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* Can use port 8094 if we need to */ + + if (match_tc_get(data->payload[0])) { + if (match_tc_reply(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_tc_get(data->payload[0])) { + if (match_tc_reply(data->payload[1], data->payload_len[1])) + return true; + } + + return false; +} + +static lpi_module_t lpi_twitcasting = { + LPI_PROTO_TWITCASTING, + LPI_CATEGORY_STREAMING, + "TwitCasting", + 25, /* Should definitely be higher than HTTP */ + match_twitcasting +}; + +void register_twitcasting(LPIModuleMap *mod_map) { + register_protocol(&lpi_twitcasting, mod_map); +} + diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index c3323f2f..70db054b 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -92,6 +92,7 @@ void register_https(LPIModuleMap *mod_map); void register_http_tunnel(LPIModuleMap *mod_map); void register_ica(LPIModuleMap *mod_map); void register_id(LPIModuleMap *mod_map); +void register_idrivesync(LPIModuleMap *mod_map); void register_imap(LPIModuleMap *mod_map); void register_imaps(LPIModuleMap *mod_map); void register_imesh(LPIModuleMap *mod_map); @@ -212,6 +213,7 @@ void register_trackmania(LPIModuleMap *mod_map); void register_trion(LPIModuleMap *mod_map); void register_trojan_win32_generic_sb(LPIModuleMap *mod_map); void register_trojan_zeroaccess(LPIModuleMap *mod_map); +void register_twitcasting(LPIModuleMap *mod_map); void register_twitch_irc(LPIModuleMap *mod_map); void register_viber(LPIModuleMap *mod_map); void register_vodlocker(LPIModuleMap *mod_map); diff --git a/lib/udp/Makefile.am b/lib/udp/Makefile.am index 5cf0d522..6c3236ad 100644 --- a/lib/udp/Makefile.am +++ b/lib/udp/Makefile.am @@ -135,6 +135,7 @@ libprotoident_udp_la_SOURCES = \ lpi_quic.cc \ lpi_radius.cc \ lpi_ramseydash.cc \ + lpi_rdp_udp.cc \ lpi_real.cc \ lpi_roblox.cc \ lpi_robocraft.cc \ diff --git a/lib/udp/lpi_baiduyun_p2p.cc b/lib/udp/lpi_baiduyun_p2p.cc index 26a844e5..596f1946 100644 --- a/lib/udp/lpi_baiduyun_p2p.cc +++ b/lib/udp/lpi_baiduyun_p2p.cc @@ -55,7 +55,9 @@ static inline bool match_byun_p2p(uint32_t payload, uint32_t len) { static inline bool match_baiduyun_p2p(lpi_data_t *data, lpi_module_t *mod UNUSED) { - if (data->server_port != 7273 && data->client_port != 7273) + if (data->server_port != 7273 && data->client_port != 7273 && + data->server_port != 7274 && + data->client_port != 7274) return false; diff --git a/lib/udp/lpi_ppstream.cc b/lib/udp/lpi_ppstream.cc index 690cf6b8..dd4fff44 100644 --- a/lib/udp/lpi_ppstream.cc +++ b/lib/udp/lpi_ppstream.cc @@ -165,6 +165,16 @@ static inline bool match_ppstream(lpi_data_t *data, lpi_module_t *mod UNUSED) { return true; } + if (match_1480_ppstream(data->payload[0], data->payload_len[0])) { + if (match_1b80_ppstream(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_1480_ppstream(data->payload[1], data->payload_len[1])) { + if (match_1b80_ppstream(data->payload[0], data->payload_len[0])) + return true; + } + if (match_8480_ppstream(data->payload[0], data->payload_len[0])) { if (match_any80_ppstream(data->payload[1])) return true; diff --git a/lib/udp/lpi_rdp_udp.cc b/lib/udp/lpi_rdp_udp.cc new file mode 100644 index 00000000..00fd7663 --- /dev/null +++ b/lib/udp/lpi_rdp_udp.cc @@ -0,0 +1,65 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + + +static inline bool match_rdp_init(uint32_t payload, uint32_t len) { + + if (len == 1232 && MATCHSTR(payload, "\xff\xff\xff\xff")) + return true; + return false; + +} + +static inline bool match_rdp_udp(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* Seen both 3389 and 4732 used as ports */ + + if (match_rdp_init(data->payload[0], data->payload_len[0])) { + if (match_rdp_init(data->payload[1], data->payload_len[1])) + return true; + } + + return false; +} + +static lpi_module_t lpi_rdp_udp = { + LPI_PROTO_UDP_RDP, + LPI_CATEGORY_REMOTE, + "RDP_UDP", + 200, + match_rdp_udp +}; + +void register_rdp_udp(LPIModuleMap *mod_map) { + register_protocol(&lpi_rdp_udp, mod_map); +} + diff --git a/lib/udp/udp_protocols.h b/lib/udp/udp_protocols.h index 463b426f..ac3b0817 100644 --- a/lib/udp/udp_protocols.h +++ b/lib/udp/udp_protocols.h @@ -164,6 +164,7 @@ void register_quake(LPIModuleMap *mod_map); void register_quic(LPIModuleMap *mod_map); void register_radius(LPIModuleMap *mod_map); void register_ramsey_dash(LPIModuleMap *mod_map); +void register_rdp_udp(LPIModuleMap *mod_map); void register_real(LPIModuleMap *mod_map); void register_roblox(LPIModuleMap *mod_map); void register_robocraft(LPIModuleMap *mod_map); From 82accb62bb67430a77ddbfb1eb82297740616f51 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Wed, 3 May 2017 17:55:12 +1200 Subject: [PATCH 31/68] Added two more game protocols Heroes of the Storm and Tom Clancy's The Division. Also made Notes RPC less prone to false positives. --- lib/libprotoident.h | 2 + lib/proto_manager.cc | 2 + lib/tcp/Makefile.am | 1 + lib/tcp/lpi_notes_rpc.cc | 5 ++- lib/tcp/lpi_thedivision.cc | 70 ++++++++++++++++++++++++++++++++++ lib/tcp/tcp_protocols.h | 1 + lib/udp/Makefile.am | 1 + lib/udp/lpi_hots.cc | 78 ++++++++++++++++++++++++++++++++++++++ lib/udp/udp_protocols.h | 1 + 9 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 lib/tcp/lpi_thedivision.cc create mode 100644 lib/udp/lpi_hots.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 74d1bd15..7194deb5 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -331,6 +331,7 @@ typedef enum { LPI_PROTO_DOUYU, LPI_PROTO_IDRIVE_SYNC, LPI_PROTO_TWITCASTING, + LPI_PROTO_THE_DIVISION, /* UDP Protocols */ LPI_PROTO_UDP, @@ -515,6 +516,7 @@ typedef enum { LPI_PROTO_UDP_TURBOVPN, LPI_PROTO_UDP_GEARSOFWAR, LPI_PROTO_UDP_RDP, + LPI_PROTO_UDP_HOTS, /* Patterns that we can match, but do not know the protocol */ LPI_PROTO_REJECTION, /* All responses are 0x02 */ diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index 2fcdf7c5..1a749d07 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -245,6 +245,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_tensafe(mod_map); register_tera(mod_map); register_tetrisonline(mod_map); + register_thedivision(mod_map); register_tip(mod_map); register_tor(mod_map); register_tpkt_generic(mod_map); @@ -354,6 +355,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_halflife(mod_map); register_hamachi_udp(mod_map); register_heroes_generals(mod_map); + register_hots(mod_map); register_icp(mod_map); register_imesh_udp(mod_map); register_ipmsg(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index 2c8b9ae0..77d049b5 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -178,6 +178,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_tensafe.cc \ lpi_tera.cc \ lpi_tetrisonline.cc \ + lpi_thedivision.cc \ lpi_tip.cc \ lpi_tor.cc \ lpi_tpkt.cc \ diff --git a/lib/tcp/lpi_notes_rpc.cc b/lib/tcp/lpi_notes_rpc.cc index 7fc00fd5..4eaff55f 100644 --- a/lib/tcp/lpi_notes_rpc.cc +++ b/lib/tcp/lpi_notes_rpc.cc @@ -40,6 +40,9 @@ static inline bool match_notes_rpc(lpi_data_t *data, lpi_module_t *mod UNUSED) { * no documented basis for this (unlike most other rules) */ + if (data->server_port != 1352 && data->client_port != 1352) + return false; + if (!match_str_either(data, "\x78\x00\x00\x00")) return false; @@ -55,7 +58,7 @@ static lpi_module_t lpi_notes_rpc = { LPI_PROTO_NOTES_RPC, LPI_CATEGORY_REMOTE, "Lotus_Notes_RPC", - 10, /* Don't really trust this rule that much :/ */ + 200, /* Don't really trust this rule that much :/ */ match_notes_rpc }; diff --git a/lib/tcp/lpi_thedivision.cc b/lib/tcp/lpi_thedivision.cc new file mode 100644 index 00000000..e80dd645 --- /dev/null +++ b/lib/tcp/lpi_thedivision.cc @@ -0,0 +1,70 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* Tom Clancy's The Division -- Ubisoft game */ + +static inline bool match_div_36(uint32_t payload, uint32_t len) { + if (len == 36 && MATCH(payload, 0x46, 0x01, 0x02, 0x20)) + return true; + return false; +} + +static inline bool match_div_8(uint32_t payload, uint32_t len) { + if (len == 8 && MATCH(payload, 0x0e, 0x01, 0x00, 0x04)) + return true; + return false; +} + +static inline bool match_thedivision(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* Port 55000? */ + + if (match_div_36(data->payload[0], data->payload_len[0])) { + if (match_div_8(data->payload[1], data->payload_len[1])) + return true; + } + + return false; +} + +static lpi_module_t lpi_thedivision = { + LPI_PROTO_THE_DIVISION, + LPI_CATEGORY_GAMING, + "Hearthstone", + 5, + match_thedivision +}; + +void register_thedivision(LPIModuleMap *mod_map) { + register_protocol(&lpi_thedivision, mod_map); +} + diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index 70db054b..567c0bf8 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -206,6 +206,7 @@ void register_tencent_games(LPIModuleMap *mod_map); void register_tensafe(LPIModuleMap *mod_map); void register_tera(LPIModuleMap *mod_map); void register_tetrisonline(LPIModuleMap *mod_map); +void register_thedivision(LPIModuleMap *mod_map); void register_tip(LPIModuleMap *mod_map); void register_tor(LPIModuleMap *mod_map); void register_tpkt_generic(LPIModuleMap *mod_map); diff --git a/lib/udp/Makefile.am b/lib/udp/Makefile.am index 6c3236ad..7fb4025f 100644 --- a/lib/udp/Makefile.am +++ b/lib/udp/Makefile.am @@ -66,6 +66,7 @@ libprotoident_udp_la_SOURCES = \ lpi_halflife.cc \ lpi_hamachi.cc \ lpi_heroes_generals.cc \ + lpi_hots.cc \ lpi_icp.cc \ lpi_imesh.cc \ lpi_ipmsg.cc \ diff --git a/lib/udp/lpi_hots.cc b/lib/udp/lpi_hots.cc new file mode 100644 index 00000000..99e6371c --- /dev/null +++ b/lib/udp/lpi_hots.cc @@ -0,0 +1,78 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* Heroes of the Storm -- Blizzard MOBA */ + +static inline bool match_hots_zero(uint32_t payload, uint32_t len) { + + if (len == 20 && MATCH(payload, 0x00, 0x00, 0x00, 0x00)) + return true; + return false; +} + +static inline bool match_hots_other(uint32_t payload, uint32_t len) { + + if (len == 20 && MATCH(payload, ANY, ANY, 0x01, 0x00)) + return true; + return false; +} + +static inline bool match_hots(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (data->server_port != 1119 && data->client_port != 1119) + return false; + + if (match_hots_zero(data->payload[0], data->payload_len[0])) { + if (match_hots_other(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_hots_zero(data->payload[0], data->payload_len[0])) { + if (match_hots_other(data->payload[1], data->payload_len[1])) + return true; + } + + return false; +} + +static lpi_module_t lpi_hots = { + LPI_PROTO_UDP_HOTS, + LPI_CATEGORY_GAMING, + "Hearthstone", + 101, + match_hots +}; + +void register_hots(LPIModuleMap *mod_map) { + register_protocol(&lpi_hots, mod_map); +} + diff --git a/lib/udp/udp_protocols.h b/lib/udp/udp_protocols.h index ac3b0817..f87d8f4b 100644 --- a/lib/udp/udp_protocols.h +++ b/lib/udp/udp_protocols.h @@ -96,6 +96,7 @@ void register_h1z1(LPIModuleMap *mod_map); void register_halflife(LPIModuleMap *mod_map); void register_hamachi_udp(LPIModuleMap *mod_map); void register_heroes_generals(LPIModuleMap *mod_map); +void register_hots(LPIModuleMap *mod_map); void register_icp(LPIModuleMap *mod_map); void register_imesh_udp(LPIModuleMap *mod_map); void register_ipmsg(LPIModuleMap *mod_map); From e8fefc24c3a8fcfb209e9bfac00d4e41ea041549 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Mon, 8 May 2017 18:05:02 +1200 Subject: [PATCH 32/68] Added 1 new protocol: Black Desert Online Updated: * Heroes of the Storm * N2Ping * Vivox Re-categorised PDBox to be "Cloud" rather than "P2PTV" --- lib/libprotoident.h | 1 + lib/proto_manager.cc | 1 + lib/tcp/Makefile.am | 1 + lib/tcp/lpi_blackdesert.cc | 83 ++++++++++++++++++++++++++++++++++++++ lib/tcp/lpi_pdbox.cc | 2 +- lib/tcp/tcp_protocols.h | 1 + lib/udp/lpi_hots.cc | 4 ++ lib/udp/lpi_n2ping.cc | 11 +++-- lib/udp/lpi_vivox.cc | 40 ++++++++++++++++++ 9 files changed, 140 insertions(+), 4 deletions(-) create mode 100644 lib/tcp/lpi_blackdesert.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 7194deb5..ced69c9d 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -332,6 +332,7 @@ typedef enum { LPI_PROTO_IDRIVE_SYNC, LPI_PROTO_TWITCASTING, LPI_PROTO_THE_DIVISION, + LPI_PROTO_BLACKDESERT, /* UDP Protocols */ LPI_PROTO_UDP, diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index 1a749d07..aa734582 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -79,6 +79,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_bitcoin(mod_map); register_bitextend(mod_map); register_bittorrent(mod_map); + register_blackdesert(mod_map); register_blizzard(mod_map); register_btsync(mod_map); register_cacaoweb(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index 77d049b5..3c6c018a 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -12,6 +12,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_bitcoin.cc \ lpi_bitextend.cc \ lpi_bittorrent.cc \ + lpi_blackdesert.cc \ lpi_blizzard.cc \ lpi_btsync.cc \ lpi_cacaoweb.cc \ diff --git a/lib/tcp/lpi_blackdesert.cc b/lib/tcp/lpi_blackdesert.cc new file mode 100644 index 00000000..1c59d783 --- /dev/null +++ b/lib/tcp/lpi_blackdesert.cc @@ -0,0 +1,83 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* First two bytes definitely look like a little-endian length + * field, so we could use that to match more reply types */ + +/* Port 9991, 9992 and 9993 */ + +static inline bool match_bdo_request(uint32_t payload, uint32_t len) { + + if (len == 111 && MATCH(payload, 0x6f, 0x00, 0x01, 0x9d)) + return true; + return false; + +} + +static inline bool match_bdo_reply(uint32_t payload, uint32_t len) { + + if (len == 112 && MATCH(payload, 0x70, 0x00, 0x01, ANY)) + return true; + + return false; + +} + +static inline bool match_blackdesert(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + + if (match_bdo_request(data->payload[0], data->payload_len[0])) { + if (match_bdo_reply(data->payload[1], data->payload_len[1])) + return true; + } + + + if (match_bdo_request(data->payload[1], data->payload_len[1])) { + if (match_bdo_reply(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_blackdesert = { + LPI_PROTO_BLACKDESERT, + LPI_CATEGORY_GAMING, + "BlackDesertOnline", + 12, + match_blackdesert +}; + +void register_blackdesert(LPIModuleMap *mod_map) { + register_protocol(&lpi_blackdesert, mod_map); +} + diff --git a/lib/tcp/lpi_pdbox.cc b/lib/tcp/lpi_pdbox.cc index db5aaf5e..d6b641fe 100644 --- a/lib/tcp/lpi_pdbox.cc +++ b/lib/tcp/lpi_pdbox.cc @@ -39,7 +39,7 @@ static inline bool match_pdbox(lpi_data_t *data, lpi_module_t *mod UNUSED) { static lpi_module_t lpi_pdbox = { LPI_PROTO_PDBOX, - LPI_CATEGORY_P2PTV, + LPI_CATEGORY_CLOUD, "PDBOX", 3, match_pdbox diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index 567c0bf8..7a052412 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -40,6 +40,7 @@ void register_badbaidu(LPIModuleMap *mod_map); void register_bitcoin(LPIModuleMap *mod_map); void register_bitextend(LPIModuleMap *mod_map); void register_bittorrent(LPIModuleMap *mod_map); +void register_blackdesert(LPIModuleMap *mod_map); void register_blizzard(LPIModuleMap *mod_map); void register_btsync(LPIModuleMap *mod_map); void register_cacaoweb(LPIModuleMap *mod_map); diff --git a/lib/udp/lpi_hots.cc b/lib/udp/lpi_hots.cc index 99e6371c..d52cb996 100644 --- a/lib/udp/lpi_hots.cc +++ b/lib/udp/lpi_hots.cc @@ -43,6 +43,10 @@ static inline bool match_hots_other(uint32_t payload, uint32_t len) { if (len == 20 && MATCH(payload, ANY, ANY, 0x01, 0x00)) return true; + if (len == 20 && MATCH(payload, ANY, ANY, 0x02, 0x00)) + return true; + if (len == 20 && MATCH(payload, ANY, ANY, 0x00, 0x00)) + return true; return false; } diff --git a/lib/udp/lpi_n2ping.cc b/lib/udp/lpi_n2ping.cc index 40a38274..ee5d7de5 100644 --- a/lib/udp/lpi_n2ping.cc +++ b/lib/udp/lpi_n2ping.cc @@ -45,11 +45,16 @@ static inline bool match_n2ping_header(uint32_t payload) { static inline bool match_n2ping(lpi_data_t *data, lpi_module_t *mod UNUSED) { - if (data->server_port != 44778 && data->client_port != 44778) { - return false; + bool validport = false; + + if (data->server_port == 44778 || data->client_port == 44778) { + validport = true; + } + if (data->server_port == 23 || data->client_port == 23) { + validport = true; } - if (match_n2ping_header(data->payload[0]) && + if (validport && match_n2ping_header(data->payload[0]) && match_n2ping_header(data->payload[1])) { if (data->payload_len[0] < 100) return true; diff --git a/lib/udp/lpi_vivox.cc b/lib/udp/lpi_vivox.cc index bb643607..57be1b26 100644 --- a/lib/udp/lpi_vivox.cc +++ b/lib/udp/lpi_vivox.cc @@ -30,8 +30,48 @@ #include "proto_manager.h" #include "proto_common.h" +static inline bool match_vivox_request(uint32_t payload, uint32_t len) { + + if (MATCH(payload, 0x80, 0x7f, 0x00, 0x00) & len == 112) + return true; + return false; + +} + +static inline bool match_vivox_reply(uint32_t payload, uint32_t len) { + + if (MATCH(payload, 0x80, 0x7f, 0x00, 0x00) & len == 99) + return true; + return false; + +} + +static inline bool match_vivox_stun(uint32_t payload, uint32_t len) { + + /* Vivox mixes a whole bunch of RTP(?) and STUN together + * when it starts up, so sometimes the STUN replies arrive + * first. */ + if (MATCH(payload, 0x01, 0x01, 0x00, 0x70) & len == 132) + return true; + return false; + +} + static inline bool match_vivox(lpi_data_t *data, lpi_module_t *mod UNUSED) { + if (match_vivox_request(data->payload[0], data->payload_len[0])) { + if (match_vivox_stun(data->payload[1], data->payload_len[1])) + return true; + if (match_vivox_reply(data->payload[1], data->payload_len[1])) + return true; + } + if (match_vivox_request(data->payload[1], data->payload_len[1])) { + if (match_vivox_stun(data->payload[0], data->payload_len[0])) + return true; + if (match_vivox_reply(data->payload[0], data->payload_len[0])) + return true; + } + /* Seen this to Vivox servers, so I'm going to make the logical * assumption */ if (!match_str_both(data, "\x80\x6f\x00\x00", "\x80\x6f\x00\x01")) From c0e0fb0288a3f81bb9c2dcedf69c5c3575756c7f Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Thu, 11 May 2017 16:56:24 +1200 Subject: [PATCH 33/68] Another batch of new protocols: * Dogecoin * Fuckcoin * ourWorld * RealVNC * Dianshijia * VPN Master Also added a few extra patterns to Heroes of the Storm --- lib/libprotoident.h | 6 +++ lib/proto_manager.cc | 6 +++ lib/tcp/Makefile.am | 4 ++ lib/tcp/lpi_dogecoin.cc | 64 +++++++++++++++++++++++++++++++ lib/tcp/lpi_fuckcoin.cc | 68 ++++++++++++++++++++++++++++++++ lib/tcp/lpi_ourworld.cc | 78 +++++++++++++++++++++++++++++++++++++ lib/tcp/lpi_realvnc.cc | 75 ++++++++++++++++++++++++++++++++++++ lib/tcp/tcp_protocols.h | 4 ++ lib/udp/Makefile.am | 2 + lib/udp/lpi_dianshijia.cc | 81 +++++++++++++++++++++++++++++++++++++++ lib/udp/lpi_hots.cc | 8 +++- lib/udp/lpi_vpnmaster.cc | 79 ++++++++++++++++++++++++++++++++++++++ lib/udp/udp_protocols.h | 2 + 13 files changed, 476 insertions(+), 1 deletion(-) create mode 100644 lib/tcp/lpi_dogecoin.cc create mode 100644 lib/tcp/lpi_fuckcoin.cc create mode 100644 lib/tcp/lpi_ourworld.cc create mode 100644 lib/tcp/lpi_realvnc.cc create mode 100644 lib/udp/lpi_dianshijia.cc create mode 100644 lib/udp/lpi_vpnmaster.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index ced69c9d..5de6ae0d 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -333,6 +333,10 @@ typedef enum { LPI_PROTO_TWITCASTING, LPI_PROTO_THE_DIVISION, LPI_PROTO_BLACKDESERT, + LPI_PROTO_REALVNC, + LPI_PROTO_DOGECOIN, + LPI_PROTO_FUCKCOIN, + LPI_PROTO_OURWORLD, /* UDP Protocols */ LPI_PROTO_UDP, @@ -518,6 +522,8 @@ typedef enum { LPI_PROTO_UDP_GEARSOFWAR, LPI_PROTO_UDP_RDP, LPI_PROTO_UDP_HOTS, + LPI_PROTO_UDP_VPNMASTER, + LPI_PROTO_UDP_DIANSHIJIA, /* Patterns that we can match, but do not know the protocol */ LPI_PROTO_REJECTION, /* All responses are 0x02 */ diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index aa734582..d4aad593 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -99,6 +99,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_dianping_tcp(mod_map); register_directconnect(mod_map); register_dns_tcp(mod_map); + register_dogecoin(mod_map); register_douyu(mod_map); register_duelingnetwork(mod_map); register_dvrns(mod_map); @@ -113,6 +114,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_fring(mod_map); register_ftpcontrol(mod_map); register_ftpdata(mod_map); + register_fuckcoin(mod_map); register_funshion_tcp(mod_map); register_gamespy_tcp(mod_map); register_git(mod_map); @@ -188,6 +190,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_tcp_no_payload(mod_map); register_omegle(mod_map); register_openvpn(mod_map); + register_ourworld(mod_map); register_palringo(mod_map); register_paltalk(mod_map); register_pandatv(mod_map); @@ -206,6 +209,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_razor(mod_map); register_rbls(mod_map); register_rdp(mod_map); + register_realvnc(mod_map); register_rejection(mod_map); register_relay(mod_map); register_revolver_nblbt(mod_map); @@ -326,6 +330,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_dht_other(mod_map); register_diablo2(mod_map); register_dianping_udp(mod_map); + register_dianshijia(mod_map); register_directconnect_udp(mod_map); register_discord(mod_map); register_dns_udp(mod_map); @@ -468,6 +473,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_ventrilo(mod_map); register_viber_udp(mod_map); register_vivox(mod_map); + register_vpnmaster(mod_map); register_vxworks_exploit(mod_map); register_warthunder(mod_map); register_wechat_udp(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index 3c6c018a..b89a4e98 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -32,6 +32,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_dianping_tcp.cc \ lpi_directconnect.cc \ lpi_dns.cc \ + lpi_dogecoin.cc \ lpi_douyu.cc \ lpi_duelingnetwork.cc \ lpi_dvrns.cc \ @@ -46,6 +47,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_fring.cc \ lpi_ftpcontrol.cc \ lpi_ftpdata.cc \ + lpi_fuckcoin.cc \ lpi_funshion.cc \ lpi_gamespy.cc \ lpi_git.cc \ @@ -121,6 +123,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_notes_rpc.cc \ lpi_omegle.cc \ lpi_openvpn.cc \ + lpi_ourworld.cc \ lpi_palringo.cc \ lpi_paltalk.cc \ lpi_pandatv.cc \ @@ -139,6 +142,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_razor.cc \ lpi_rbls.cc \ lpi_rdp.cc \ + lpi_realvnc.cc \ lpi_rejection.cc \ lpi_relay.cc \ lpi_revolver_nblbt.cc \ diff --git a/lib/tcp/lpi_dogecoin.cc b/lib/tcp/lpi_dogecoin.cc new file mode 100644 index 00000000..84188870 --- /dev/null +++ b/lib/tcp/lpi_dogecoin.cc @@ -0,0 +1,64 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_dc_magic(uint32_t payload, uint32_t len) { + + if (len == 129 && MATCH(payload, 0xc0, 0xc0, 0xc0, 0xc0)) + return true; + return false; + +} + +static inline bool match_dogecoin(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* Port == 22556 */ + + if (match_dc_magic(data->payload[0], data->payload_len[0])) { + if (match_dc_magic(data->payload[1], data->payload_len[1])) + return true; + } + + return false; +} + +static lpi_module_t lpi_dogecoin = { + LPI_PROTO_DOGECOIN, + LPI_CATEGORY_ECOMMERCE, + "Dogecoin", + 8, + match_dogecoin +}; + +void register_dogecoin(LPIModuleMap *mod_map) { + register_protocol(&lpi_dogecoin, mod_map); +} + diff --git a/lib/tcp/lpi_fuckcoin.cc b/lib/tcp/lpi_fuckcoin.cc new file mode 100644 index 00000000..e407d1c1 --- /dev/null +++ b/lib/tcp/lpi_fuckcoin.cc @@ -0,0 +1,68 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_fc_magic(uint32_t payload, uint32_t len) { + + if (MATCH(payload, 0xfb, 0xc0, 0xb6, 0xdb)) { + if (len == 126) + return true; + if (len == 146) + return true; + } + return false; + +} + +static inline bool match_fuckcoin(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* Port == 9333 */ + + if (match_fc_magic(data->payload[0], data->payload_len[0])) { + if (match_fc_magic(data->payload[1], data->payload_len[1])) + return true; + } + + return false; +} + +static lpi_module_t lpi_fuckcoin = { + LPI_PROTO_FUCKCOIN, + LPI_CATEGORY_ECOMMERCE, + "Dogecoin", + 8, + match_fuckcoin +}; + +void register_fuckcoin(LPIModuleMap *mod_map) { + register_protocol(&lpi_fuckcoin, mod_map); +} + diff --git a/lib/tcp/lpi_ourworld.cc b/lib/tcp/lpi_ourworld.cc new file mode 100644 index 00000000..650b8ebc --- /dev/null +++ b/lib/tcp/lpi_ourworld.cc @@ -0,0 +1,78 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* ourworld.com -- Flash-based MMO / game site combo thing */ + +static inline bool match_custom_flash_36(uint32_t payload, uint32_t len) { + + if (len == 36 && MATCH(payload, '<', 'm', '>', 'A')) + return true; + return false; +} + +static inline bool match_custom_flash_200(uint32_t payload, uint32_t len) { + + if (MATCH(payload, '<', 'm', '>', 'A')) { + if (len >= 200 && len <= 250) + return true; + } + return false; +} + +static inline bool match_ourworld(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* Port 9310 */ + if (match_custom_flash_36(data->payload[0], data->payload_len[0])) { + if (match_custom_flash_200(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_custom_flash_36(data->payload[1], data->payload_len[1])) { + if (match_custom_flash_200(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_ourworld = { + LPI_PROTO_OURWORLD, + LPI_CATEGORY_GAMING, + "OurWorld", + 5, + match_ourworld +}; + +void register_ourworld(LPIModuleMap *mod_map) { + register_protocol(&lpi_ourworld, mod_map); +} + diff --git a/lib/tcp/lpi_realvnc.cc b/lib/tcp/lpi_realvnc.cc new file mode 100644 index 00000000..77ca4ecc --- /dev/null +++ b/lib/tcp/lpi_realvnc.cc @@ -0,0 +1,75 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_real(uint32_t payload) { + if (MATCH(payload, 'R', 'E', 'A', 'L')) + return true; + return false; +} + +static inline bool match_real_resp(uint32_t payload, uint32_t len) { + + /* Byte 3 could be a length field (len - 3). */ + if (MATCH(payload, 0x81, 0x00, 0x14, ANY) && len == 23) + return true; + return false; +} + + +static inline bool match_realvnc(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + + if (match_real(data->payload[0])) { + if (match_real_resp(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_real(data->payload[1])) { + if (match_real_resp(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_realvnc = { + LPI_PROTO_REALVNC, + LPI_CATEGORY_REMOTE, + "RealVNC", + 10, + match_realvnc +}; + +void register_realvnc(LPIModuleMap *mod_map) { + register_protocol(&lpi_realvnc, mod_map); +} + diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index 7a052412..f95cd943 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -60,6 +60,7 @@ void register_diablo3(LPIModuleMap *mod_map); void register_dianping_tcp(LPIModuleMap *mod_map); void register_directconnect(LPIModuleMap *mod_map); void register_dns_tcp(LPIModuleMap *mod_map); +void register_dogecoin(LPIModuleMap *mod_map); void register_douyu(LPIModuleMap *mod_map); void register_duelingnetwork(LPIModuleMap *mod_map); void register_dvrns(LPIModuleMap *mod_map); @@ -74,6 +75,7 @@ void register_flash(LPIModuleMap *mod_map); void register_fring(LPIModuleMap *mod_map); void register_ftpcontrol(LPIModuleMap *mod_map); void register_ftpdata(LPIModuleMap *mod_map); +void register_fuckcoin(LPIModuleMap *mod_map); void register_funshion_tcp(LPIModuleMap *mod_map); void register_gamespy_tcp(LPIModuleMap *mod_map); void register_git(LPIModuleMap *mod_map); @@ -149,6 +151,7 @@ void register_notes_rpc(LPIModuleMap *mod_map); void register_tcp_no_payload(LPIModuleMap *mod_map); void register_omegle(LPIModuleMap *mod_map); void register_openvpn(LPIModuleMap *mod_map); +void register_ourworld(LPIModuleMap *mod_map); void register_palringo(LPIModuleMap *mod_map); void register_paltalk(LPIModuleMap *mod_map); void register_pandatv(LPIModuleMap *mod_map); @@ -167,6 +170,7 @@ void register_qvod(LPIModuleMap *mod_map); void register_razor(LPIModuleMap *mod_map); void register_rbls(LPIModuleMap *mod_map); void register_rdp(LPIModuleMap *mod_map); +void register_realvnc(LPIModuleMap *mod_map); void register_rejection(LPIModuleMap *mod_map); void register_relay(LPIModuleMap *mod_map); void register_revolver_nblbt(LPIModuleMap *mod_map); diff --git a/lib/udp/Makefile.am b/lib/udp/Makefile.am index 7fb4025f..8810a0b0 100644 --- a/lib/udp/Makefile.am +++ b/lib/udp/Makefile.am @@ -36,6 +36,7 @@ libprotoident_udp_la_SOURCES = \ lpi_dht_other.cc \ lpi_diablo2.cc \ lpi_dianping_udp.cc \ + lpi_dianshijia.cc \ lpi_directconnect.cc \ lpi_discord.cc \ lpi_dns.cc \ @@ -178,6 +179,7 @@ libprotoident_udp_la_SOURCES = \ lpi_ventrilo.cc \ lpi_viber.cc \ lpi_vivox.cc \ + lpi_vpnmaster.cc \ lpi_vxworks_exploit.cc \ lpi_warthunder.cc \ lpi_wechat_udp.cc \ diff --git a/lib/udp/lpi_dianshijia.cc b/lib/udp/lpi_dianshijia.cc new file mode 100644 index 00000000..74c395ef --- /dev/null +++ b/lib/udp/lpi_dianshijia.cc @@ -0,0 +1,81 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + + +/* APK name = com.dianshijia.tvlive */ +/* Published by New Zealand ibestv Trading Limited (?) */ + +static inline bool match_dsj_req(uint32_t payload, uint32_t len) { + if (len == 20 && MATCHSTR(payload, "\xff\x03\x00\x00")) + return true; + + return false; +} + +static inline bool match_dsj_reply(uint32_t payload, uint32_t len) { + if (len == 20 && MATCHSTR(payload, "\xff\x05\x00\x00")) + return true; + if (len == 20 && MATCHSTR(payload, "\xff\x03\x01\x00")) + return true; + + return false; +} + +static inline bool match_dianshijia(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + + if (match_dsj_req(data->payload[0], data->payload_len[0])) { + if (match_dsj_reply(data->payload[1], data->payload_len[1])) + return true; + } + + + if (match_dsj_req(data->payload[1], data->payload_len[1])) { + if (match_dsj_reply(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_dianshijia = { + LPI_PROTO_UDP_DIANSHIJIA, + LPI_CATEGORY_P2PTV, /* I think... */ + "Dianshijia", + 100, + match_dianshijia +}; + +void register_dianshijia(LPIModuleMap *mod_map) { + register_protocol(&lpi_dianshijia, mod_map); +} + diff --git a/lib/udp/lpi_hots.cc b/lib/udp/lpi_hots.cc index d52cb996..0689ca81 100644 --- a/lib/udp/lpi_hots.cc +++ b/lib/udp/lpi_hots.cc @@ -45,8 +45,14 @@ static inline bool match_hots_other(uint32_t payload, uint32_t len) { return true; if (len == 20 && MATCH(payload, ANY, ANY, 0x02, 0x00)) return true; - if (len == 20 && MATCH(payload, ANY, ANY, 0x00, 0x00)) + if (len == 20 && MATCH(payload, ANY, ANY, 0x03, 0x00)) return true; + if (len == 20 && MATCH(payload, ANY, ANY, 0x00, 0x00)) { + if (MATCH(payload, 0x00, 0x00, 0x00, 0x00)) + return false; + return true; + } + return false; } diff --git a/lib/udp/lpi_vpnmaster.cc b/lib/udp/lpi_vpnmaster.cc new file mode 100644 index 00000000..df7c4c9c --- /dev/null +++ b/lib/udp/lpi_vpnmaster.cc @@ -0,0 +1,79 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* VPN Proxy Master-Free security -- free android VPN via China */ + +static inline bool match_ce7(uint32_t payload, uint32_t len) { + + if (len == 30 && MATCH(payload, 'C', 'E', '7', 'E')) + return true; + if (len == 29 && MATCH(payload, 'C', 'E', '7', 0x05)) + return true; + return false; + +} + +static inline bool match_7b(uint32_t payload, uint32_t len) { + + if (len == 14 && MATCH(payload, 0x7b, ANY, ANY, ANY)) + return true; + return false; + +} + +static inline bool match_vpnmaster(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_ce7(data->payload[0], data->payload_len[0])) { + if (match_7b(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_ce7(data->payload[0], data->payload_len[0])) { + if (match_7b(data->payload[1], data->payload_len[1])) + return true; + } + + return false; +} + +static lpi_module_t lpi_vpnmaster = { + LPI_PROTO_UDP_VPNMASTER, + LPI_CATEGORY_TUNNELLING, + "VPNMaster", + 5, + match_vpnmaster +}; + +void register_vpnmaster(LPIModuleMap *mod_map) { + register_protocol(&lpi_vpnmaster, mod_map); +} + diff --git a/lib/udp/udp_protocols.h b/lib/udp/udp_protocols.h index f87d8f4b..a2e28a86 100644 --- a/lib/udp/udp_protocols.h +++ b/lib/udp/udp_protocols.h @@ -66,6 +66,7 @@ void register_dht_dict(LPIModuleMap *mod_map); void register_dht_other(LPIModuleMap *mod_map); void register_diablo2(LPIModuleMap *mod_map); void register_dianping_udp(LPIModuleMap *mod_map); +void register_dianshijia(LPIModuleMap *mod_map); void register_directconnect_udp(LPIModuleMap *mod_map); void register_discord(LPIModuleMap *mod_map); void register_dns_udp(LPIModuleMap *mod_map); @@ -208,6 +209,7 @@ void register_unreal(LPIModuleMap *mod_map); void register_ventrilo(LPIModuleMap *mod_map); void register_viber_udp(LPIModuleMap *mod_map); void register_vivox(LPIModuleMap *mod_map); +void register_vpnmaster(LPIModuleMap *mod_map); void register_vxworks_exploit(LPIModuleMap *mod_map); void register_warthunder(LPIModuleMap *mod_map); void register_wechat_udp(LPIModuleMap *mod_map); From 29160e5f14695dcae3333bed2e744271882e49c9 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Tue, 16 May 2017 13:13:01 +1200 Subject: [PATCH 34/68] Added rule for PS4 Remote Play Other updated protocols: * Heroes of the Storm -- include port 3724 and 0x04 pattern * Maplestory China -- add 0x8f pattern * NTP Reflection -- recognise spoofed responses * Minecraft -- support any protocol version --- lib/libprotoident.h | 1 + lib/proto_manager.cc | 1 + lib/tcp/lpi_maplestory_china.cc | 2 + lib/tcp/lpi_minecraft.cc | 8 +--- lib/udp/Makefile.am | 1 + lib/udp/lpi_hots.cc | 11 ++++- lib/udp/lpi_ntp_reflect.cc | 9 +++++ lib/udp/lpi_ps4_remoteplay.cc | 72 +++++++++++++++++++++++++++++++++ lib/udp/udp_protocols.h | 1 + 9 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 lib/udp/lpi_ps4_remoteplay.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 5de6ae0d..14ff036d 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -524,6 +524,7 @@ typedef enum { LPI_PROTO_UDP_HOTS, LPI_PROTO_UDP_VPNMASTER, LPI_PROTO_UDP_DIANSHIJIA, + LPI_PROTO_UDP_PS4_REMOTEPLAY, /* Patterns that we can match, but do not know the protocol */ LPI_PROTO_REJECTION, /* All responses are 0x02 */ diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index d4aad593..074f6755 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -420,6 +420,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_pplive(mod_map); register_ppstream(mod_map); //register_probable_gnutella(mod_map); + register_ps4_remoteplay(mod_map); register_psn(mod_map); register_punkbuster(mod_map); register_pyzor(mod_map); diff --git a/lib/tcp/lpi_maplestory_china.cc b/lib/tcp/lpi_maplestory_china.cc index 07f0dc3a..8bc7dfe0 100644 --- a/lib/tcp/lpi_maplestory_china.cc +++ b/lib/tcp/lpi_maplestory_china.cc @@ -37,6 +37,8 @@ static inline bool match_cms_hello(uint32_t payload, uint32_t len) { return true; if (MATCH(payload, 0x0e, 0x00, 0x8e, 0x00)) return true; + if (MATCH(payload, 0x0e, 0x00, 0x8f, 0x00)) + return true; } return false; diff --git a/lib/tcp/lpi_minecraft.cc b/lib/tcp/lpi_minecraft.cc index aae463ee..d84deeb6 100644 --- a/lib/tcp/lpi_minecraft.cc +++ b/lib/tcp/lpi_minecraft.cc @@ -78,11 +78,7 @@ static inline bool match_mc_handshake(uint32_t payload, uint32_t len) { replen = ntohl(payload) >> 24; if (replen == len - 1) { - /* Older protocol version */ - if (MATCH(payload, ANY, 0x00, 0x2f, ANY)) - return true; - /* Most recent protocol version (02/08/16) */ - if (MATCH(payload, ANY, 0x00, 0xd2, ANY)) + if (MATCH(payload, ANY, 0x00, ANY, ANY)) return true; } @@ -143,7 +139,7 @@ static lpi_module_t lpi_minecraft = { LPI_PROTO_MINECRAFT, LPI_CATEGORY_GAMING, "Minecraft", - 5, + 35, match_minecraft }; diff --git a/lib/udp/Makefile.am b/lib/udp/Makefile.am index 8810a0b0..459b9a7c 100644 --- a/lib/udp/Makefile.am +++ b/lib/udp/Makefile.am @@ -127,6 +127,7 @@ libprotoident_udp_la_SOURCES = \ lpi_pplive.cc \ lpi_ppstream.cc \ lpi_probable_gnutella.cc \ + lpi_ps4_remoteplay.cc \ lpi_psn.cc \ lpi_punkbuster.cc \ lpi_pyzor.cc \ diff --git a/lib/udp/lpi_hots.cc b/lib/udp/lpi_hots.cc index 0689ca81..47b37a45 100644 --- a/lib/udp/lpi_hots.cc +++ b/lib/udp/lpi_hots.cc @@ -47,6 +47,8 @@ static inline bool match_hots_other(uint32_t payload, uint32_t len) { return true; if (len == 20 && MATCH(payload, ANY, ANY, 0x03, 0x00)) return true; + if (len == 20 && MATCH(payload, ANY, ANY, 0x04, 0x00)) + return true; if (len == 20 && MATCH(payload, ANY, ANY, 0x00, 0x00)) { if (MATCH(payload, 0x00, 0x00, 0x00, 0x00)) return false; @@ -58,7 +60,14 @@ static inline bool match_hots_other(uint32_t payload, uint32_t len) { static inline bool match_hots(lpi_data_t *data, lpi_module_t *mod UNUSED) { - if (data->server_port != 1119 && data->client_port != 1119) + bool validport = false; + + if (data->server_port == 1119 || data->client_port == 1119) + validport = true; + if (data->server_port == 3724 || data->client_port == 3724) + validport = true; + + if (!validport) return false; if (match_hots_zero(data->payload[0], data->payload_len[0])) { diff --git a/lib/udp/lpi_ntp_reflect.cc b/lib/udp/lpi_ntp_reflect.cc index e076b810..21e2df02 100644 --- a/lib/udp/lpi_ntp_reflect.cc +++ b/lib/udp/lpi_ntp_reflect.cc @@ -33,6 +33,9 @@ static inline bool match_monlist(uint32_t payload) { + if (len == 0) + return true; + if (MATCH(payload, 0x17, 0x00, 0x03, 0x2a)) return true; return false; @@ -48,10 +51,16 @@ static inline bool match_monlist_reply(uint32_t payload, uint32_t len) { /* NTPv2 reply */ if (MATCH(payload, 0x97, 0x00, 0x03, 0x2a)) return true; + if (MATCH(payload, 0xd7, 0x00, 0x03, 0x2a)) + return true; /* NTPv3 reply */ if (MATCH(payload, 0x9f, 0x00, 0x03, 0x2a)) return true; + if (MATCH(payload, 0xdf, 0x00, 0x03, 0x2a)) + return true; + + return false; diff --git a/lib/udp/lpi_ps4_remoteplay.cc b/lib/udp/lpi_ps4_remoteplay.cc new file mode 100644 index 00000000..748a57d7 --- /dev/null +++ b/lib/udp/lpi_ps4_remoteplay.cc @@ -0,0 +1,72 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool valid_port(uint16_t porta, uint16_t portb) { + + if (porta == 9296 || portb == 9296) + return true; + if (porta == 9297 || portb == 9297) + return true; + + return false; +} + +static inline bool match_ps4_remoteplay(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (!valid_port(data->server_port, data->client_port)) + return false; + + /* Examples that I have are 88 bytes, but this probably depends on + * lengths of user and device names */ + if (data->payload_len[0] != data->payload_len[1]) + return false; + + if (MATCH(data->payload[0], 0x01, 0x00, 0x00, 0x00)) { + if (MATCH(data->payload[1], 0x01, 0x00, 0x00, 0x00)) + return true; + } + + return false; +} + +static lpi_module_t lpi_ps4_remoteplay = { + LPI_PROTO_UDP_PS4_REMOTEPLAY, + LPI_CATEGORY_GAMING, + "PS4_RemotePlay", + 150, + match_ps4_remoteplay +}; + +void register_ps4_remoteplay(LPIModuleMap *mod_map) { + register_protocol(&lpi_ps4_remoteplay, mod_map); +} + diff --git a/lib/udp/udp_protocols.h b/lib/udp/udp_protocols.h index a2e28a86..2a838201 100644 --- a/lib/udp/udp_protocols.h +++ b/lib/udp/udp_protocols.h @@ -156,6 +156,7 @@ void register_portmap_rpc(LPIModuleMap *mod_map); void register_pplive(LPIModuleMap *mod_map); void register_ppstream(LPIModuleMap *mod_map); void register_probable_gnutella(LPIModuleMap *mod_map); +void register_ps4_remoteplay(LPIModuleMap *mod_map); void register_psn(LPIModuleMap *mod_map); void register_punkbuster(LPIModuleMap *mod_map); void register_pyzor(LPIModuleMap *mod_map); From 5809e3e87acd61b14b8d691c010536e59c835529 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Wed, 17 May 2017 11:55:08 +1200 Subject: [PATCH 35/68] Add missing len parameter in lpi_ntp_reflect.cc --- lib/udp/lpi_ntp_reflect.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/udp/lpi_ntp_reflect.cc b/lib/udp/lpi_ntp_reflect.cc index 21e2df02..92502f94 100644 --- a/lib/udp/lpi_ntp_reflect.cc +++ b/lib/udp/lpi_ntp_reflect.cc @@ -31,7 +31,7 @@ #include "proto_common.h" -static inline bool match_monlist(uint32_t payload) { +static inline bool match_monlist(uint32_t payload, uint32_t len) { if (len == 0) return true; @@ -71,12 +71,12 @@ static inline bool match_ntp_reflect(lpi_data_t *data, lpi_module_t *mod UNUSED) if (data->server_port != 123 && data->client_port != 123) return false; - if (match_monlist(data->payload[0])) { + if (match_monlist(data->payload[0], data->payload_len[0])) { if (match_monlist_reply(data->payload[1], data->payload_len[1])) return true; } - if (match_monlist(data->payload[1])) { + if (match_monlist(data->payload[1], data->payload_len[1])) { if (match_monlist_reply(data->payload[0], data->payload_len[0])) return true; } From 66423170bbd6ce2b22e7703f8da01dc42a4030d9 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Wed, 17 May 2017 11:56:07 +1200 Subject: [PATCH 36/68] Re-categorise 360.cn as "Security" Still not 100% sure what it is, but I think the Zegost references are a misinterpretation. Zegost does resolve 360.cn names, but that is more likely so it can block those IPs. --- lib/udp/lpi_360cn.cc | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/udp/lpi_360cn.cc b/lib/udp/lpi_360cn.cc index 9c99310b..e0e631b9 100644 --- a/lib/udp/lpi_360cn.cc +++ b/lib/udp/lpi_360cn.cc @@ -33,11 +33,7 @@ /* Not 100% sure what this is but: * - it's on port 53 but is definitely not DNS * - involves servers owned by 360.cn, who 'supposedly' are antivirus experts - * - most of these same servers are contacted by various known malware, - particularly Zegost variants * - the protocol appears to be a custom encryption protocol - * - * In all likelihood, this is the C&C protocol for some nasty Chinese backdoor. */ @@ -88,7 +84,7 @@ static inline bool match_360cn(lpi_data_t *data, lpi_module_t *mod UNUSED) { static lpi_module_t lpi_360cn = { LPI_PROTO_UDP_360CN, - LPI_CATEGORY_MALWARE, + LPI_CATEGORY_SECURITY, "360.cn", 50, match_360cn From 3a535b51613be962fc9ccde39c79981ba39cbd72 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Wed, 17 May 2017 11:57:28 +1200 Subject: [PATCH 37/68] Add another Facebook Zero pattern --- lib/tcp/lpi_zero.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/tcp/lpi_zero.cc b/lib/tcp/lpi_zero.cc index b9f8805c..3904a255 100644 --- a/lib/tcp/lpi_zero.cc +++ b/lib/tcp/lpi_zero.cc @@ -57,6 +57,8 @@ static inline bool match_zero_fb_shlo(uint32_t payload, uint32_t len) { return true; if (MATCH(payload, 0x30, 0x11, 0x0f, 0x00)) return true; + if (MATCH(payload, 0x30, 0x14, 0x05, 0x00)) + return true; if (MATCH(payload, 0x30, ANY, 0x00, 0x00)) return true; return false; From 1206267227249674afdf625d4c556fbf01bdd387 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Wed, 17 May 2017 11:57:43 +1200 Subject: [PATCH 38/68] Add new protocol: Graal Online Era --- lib/libprotoident.h | 1 + lib/proto_manager.cc | 1 + lib/tcp/Makefile.am | 1 + lib/tcp/lpi_graalonlineera.cc | 82 +++++++++++++++++++++++++++++++++++ lib/tcp/tcp_protocols.h | 1 + 5 files changed, 86 insertions(+) create mode 100644 lib/tcp/lpi_graalonlineera.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 14ff036d..64f4f19d 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -337,6 +337,7 @@ typedef enum { LPI_PROTO_DOGECOIN, LPI_PROTO_FUCKCOIN, LPI_PROTO_OURWORLD, + LPI_PROTO_GRAAL_ONLINE_ERA, /* UDP Protocols */ LPI_PROTO_UDP, diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index 074f6755..94285ddc 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -122,6 +122,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_gnutella(mod_map); register_goku(mod_map); register_googlehangouts(mod_map); + register_graalonlineera(mod_map); register_guildwars2(mod_map); register_hamachi(mod_map); register_harveys(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index b89a4e98..d7dc8df2 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -55,6 +55,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_gnutella.cc \ lpi_goku.cc \ lpi_googlehangouts.cc \ + lpi_graalonlineera.cc \ lpi_guildwars2.cc \ lpi_hamachi.cc \ lpi_harveys.cc \ diff --git a/lib/tcp/lpi_graalonlineera.cc b/lib/tcp/lpi_graalonlineera.cc new file mode 100644 index 00000000..44a3ce40 --- /dev/null +++ b/lib/tcp/lpi_graalonlineera.cc @@ -0,0 +1,82 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_goe_gnp(uint32_t payload, uint32_t len) { + if (len == 8 && MATCH(payload, 'G', 'N', 'P', '1')) + return true; + return false; +} + +static inline bool match_goe_binary(uint32_t payload, uint32_t len) { + + if (len >= 275 && len <= 300) { + if (MATCH(payload, 0x01, 0x02, 0x00, 0x01)) + return true; + if (MATCH(payload, 0x01, 0x03, 0x00, 0x01)) + return true; + + } + return false; + +} + + +static inline bool match_graalonlineera(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* Port 14900 */ + + if (match_goe_gnp(data->payload[0], data->payload_len[0])) { + if (match_goe_binary(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_goe_gnp(data->payload[1], data->payload_len[1])) { + if (match_goe_binary(data->payload[0], data->payload_len[0])) + return true; + } + + + return false; +} + +static lpi_module_t lpi_graalonlineera = { + LPI_PROTO_GRAAL_ONLINE_ERA, + LPI_CATEGORY_GAMING, + "GraalOnlineEra", + 8, + match_graalonlineera +}; + +void register_graalonlineera(LPIModuleMap *mod_map) { + register_protocol(&lpi_graalonlineera, mod_map); +} + diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index f95cd943..33dda8e8 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -83,6 +83,7 @@ void register_glupteba(LPIModuleMap *mod_map); void register_gnutella(LPIModuleMap *mod_map); void register_goku(LPIModuleMap *mod_map); void register_googlehangouts(LPIModuleMap *mod_map); +void register_graalonlineera(LPIModuleMap *mod_map); void register_guildwars2(LPIModuleMap *mod_map); void register_hamachi(LPIModuleMap *mod_map); void register_harveys(LPIModuleMap *mod_map); From f36a9219db84be7731fdf300361bfad805d228c3 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 19 May 2017 16:49:27 +1200 Subject: [PATCH 39/68] Tried to simplify PPstream rules This will allow us to match the increasing number of variants, although there is a slight chance of increased FPs. Tweaked priority to help compensate. --- lib/udp/lpi_ppstream.cc | 91 +++++++++++------------------------------ 1 file changed, 24 insertions(+), 67 deletions(-) diff --git a/lib/udp/lpi_ppstream.cc b/lib/udp/lpi_ppstream.cc index dd4fff44..054c5e8f 100644 --- a/lib/udp/lpi_ppstream.cc +++ b/lib/udp/lpi_ppstream.cc @@ -95,102 +95,59 @@ static inline bool match_ppstream_payload(uint32_t payload, uint32_t len) { return false; } +static inline bool match_8480_ppstream(uint32_t payload, uint32_t len) { + + + if (len == 132 && MATCH(payload, 0x84, 0x80, 0xc0, 0xd1)) + return true; + if (len == 132 && MATCH(payload, 0x84, 0x80, 0xd1, 0xc0)) + return true; -static inline bool match_1580_ppstream(uint32_t payload, uint32_t len) { - if (len == 24) { - if (MATCH(payload, 0x15, 0x80, 0x51, 0x10)) - return true; - if (MATCH(payload, 0x15, 0x80, 0x40, 0x01)) - return true; - } return false; } -static inline bool match_1480_ppstream(uint32_t payload, uint32_t len) { - if (len == 24) { - if (MATCH(payload, 0x14, 0x80, 0x50, 0x11)) - return true; - } - return false; +static inline bool match_80_ppstream(uint32_t payload, uint32_t len) { -} + uint32_t hlen = ntohl(payload) >> 24; -static inline bool match_1b80_ppstream(uint32_t payload, uint32_t len) { - if (len == 27) { - if (MATCH(payload, 0x1b, 0x80, ANY, ANY)) + if (MATCH(payload, ANY, 0x80, ANY, ANY)) { + if (len == hlen ) return true; - } - return false; -} -static inline bool match_8480_ppstream(uint32_t payload, uint32_t len) { - if (len == 132) { - if (MATCH(payload, 0x84, 0x80, 0xc0, 0xd1)) - return true; - if (MATCH(payload, 0x84, 0x80, 0xd1, 0xc0)) + /* There must be a minimum datagram size */ + if (len == 24 && hlen < 24) return true; } + return false; -} -static inline bool match_any80_ppstream(uint32_t payload) { - if (MATCH(payload, ANY, 0x80, ANY, ANY)) - return true; - return false; } static inline bool match_ppstream(lpi_data_t *data, lpi_module_t *mod UNUSED) { - if (match_1580_ppstream(data->payload[0], data->payload_len[0])) { - if (match_1b80_ppstream(data->payload[1], data->payload_len[1])) - return true; - if (match_1580_ppstream(data->payload[1], data->payload_len[1])) - return true; - if (match_1480_ppstream(data->payload[1], data->payload_len[1])) - return true; - if (match_any80_ppstream(data->payload[1])) - return true; - } - - if (match_1580_ppstream(data->payload[1], data->payload_len[1])) { - if (match_1b80_ppstream(data->payload[0], data->payload_len[0])) - return true; - if (match_1580_ppstream(data->payload[0], data->payload_len[0])) - return true; - if (match_1480_ppstream(data->payload[0], data->payload_len[0])) - return true; - if (match_any80_ppstream(data->payload[0])) - return true; - } - - if (match_1480_ppstream(data->payload[0], data->payload_len[0])) { - if (match_1b80_ppstream(data->payload[1], data->payload_len[1])) - return true; - } - - if (match_1480_ppstream(data->payload[1], data->payload_len[1])) { - if (match_1b80_ppstream(data->payload[0], data->payload_len[0])) + if (match_ppstream_payload(data->payload[0], data->payload_len[0])) { + if (match_ppstream_payload(data->payload[1], data->payload_len[1])) return true; } if (match_8480_ppstream(data->payload[0], data->payload_len[0])) { - if (match_any80_ppstream(data->payload[1])) + if (MATCH(data->payload[1], ANY, 0x80, ANY, ANY)) return true; } if (match_8480_ppstream(data->payload[1], data->payload_len[1])) { - if (match_any80_ppstream(data->payload[0])) + if (MATCH(data->payload[0], ANY, 0x80, ANY, ANY)) return true; } - if (!match_ppstream_payload(data->payload[0], data->payload_len[0])) - return false; - if (!match_ppstream_payload(data->payload[1], data->payload_len[1])) - return false; + if (match_80_ppstream(data->payload[0], data->payload_len[0])) { + if (match_80_ppstream(data->payload[1], data->payload_len[1])) + return true; + } - return true; + return false; } @@ -199,7 +156,7 @@ static lpi_module_t lpi_ppstream = { LPI_PROTO_UDP_PPSTREAM, LPI_CATEGORY_P2PTV, "PPStream", - 5, + 150, match_ppstream }; From e22772e8b72fe7d6060eac62e8c41e082f205621 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 19 May 2017 16:50:18 +1200 Subject: [PATCH 40/68] Fixed copy-paste bug in lpi_hots.cc --- lib/udp/lpi_hots.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/udp/lpi_hots.cc b/lib/udp/lpi_hots.cc index 47b37a45..d94710ae 100644 --- a/lib/udp/lpi_hots.cc +++ b/lib/udp/lpi_hots.cc @@ -75,8 +75,8 @@ static inline bool match_hots(lpi_data_t *data, lpi_module_t *mod UNUSED) { return true; } - if (match_hots_zero(data->payload[0], data->payload_len[0])) { - if (match_hots_other(data->payload[1], data->payload_len[1])) + if (match_hots_zero(data->payload[1], data->payload_len[1])) { + if (match_hots_other(data->payload[0], data->payload_len[0])) return true; } From f93957f2fe2beabaa35d319331985e8be6544130 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 19 May 2017 16:50:51 +1200 Subject: [PATCH 41/68] Updated Viber rule to match video calls and other large flows. --- lib/udp/lpi_viber.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/udp/lpi_viber.cc b/lib/udp/lpi_viber.cc index a974e62d..68ff8802 100644 --- a/lib/udp/lpi_viber.cc +++ b/lib/udp/lpi_viber.cc @@ -33,7 +33,7 @@ /* Thanks to Remy Mudingay for providing traces to identify this protocol */ static inline bool match_viber_request(uint32_t payload, uint32_t plen) { - if (plen != 12) + if (plen != 12 && plen != 16) return false; if (MATCH(payload, ANY, ANY, 0x03, 0x00)) return true; @@ -82,11 +82,15 @@ static inline bool match_viber_udp(lpi_data_t *data, lpi_module_t *mod UNUSED) { if (match_viber_request(data->payload[0], data->payload_len[0])) { if (match_viber_response(data->payload[1], data->payload_len[1])) return true; + if (match_viber_request(data->payload[1], data->payload_len[1])) + return true; } if (match_viber_request(data->payload[1], data->payload_len[1])) { if (match_viber_response(data->payload[0], data->payload_len[0])) return true; + if (match_viber_request(data->payload[0], data->payload_len[0])) + return true; } /* This targets the viber flow that goes to the AWS server */ From 63ce89b7fa4b3788aac5a117e8989dc08ad38346 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Tue, 23 May 2017 10:53:13 +1200 Subject: [PATCH 42/68] Added 2 more protocols: Webex and Star Citizen Also added yet another pattern to HOTS --- lib/libprotoident.h | 2 + lib/proto_manager.cc | 2 + lib/udp/Makefile.am | 2 + lib/udp/lpi_hots.cc | 2 + lib/udp/lpi_starcitizen.cc | 78 ++++++++++++++++++++++++++++++++++++++ lib/udp/lpi_webex.cc | 78 ++++++++++++++++++++++++++++++++++++++ lib/udp/udp_protocols.h | 2 + 7 files changed, 166 insertions(+) create mode 100644 lib/udp/lpi_starcitizen.cc create mode 100644 lib/udp/lpi_webex.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 64f4f19d..ec1b1146 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -526,6 +526,8 @@ typedef enum { LPI_PROTO_UDP_VPNMASTER, LPI_PROTO_UDP_DIANSHIJIA, LPI_PROTO_UDP_PS4_REMOTEPLAY, + LPI_PROTO_UDP_STARCITIZEN, + LPI_PROTO_UDP_WEBEX, /* Patterns that we can match, but do not know the protocol */ LPI_PROTO_REJECTION, /* All responses are 0x02 */ diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index 94285ddc..e045fc2e 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -451,6 +451,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_spotify_bcast(mod_map); register_sql_worm(mod_map); register_ssdp(mod_map); + register_starcitizen(mod_map); register_starcraft(mod_map); register_steamfriends(mod_map); register_steam_localbroadcast(mod_map); @@ -478,6 +479,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_vpnmaster(mod_map); register_vxworks_exploit(mod_map); register_warthunder(mod_map); + register_webex(mod_map); register_wechat_udp(mod_map); register_winmessage(mod_map); register_wolfet(mod_map); diff --git a/lib/udp/Makefile.am b/lib/udp/Makefile.am index 459b9a7c..5850548c 100644 --- a/lib/udp/Makefile.am +++ b/lib/udp/Makefile.am @@ -157,6 +157,7 @@ libprotoident_udp_la_SOURCES = \ lpi_spotify_bcast.cc \ lpi_sql_worm.cc \ lpi_ssdp.cc \ + lpi_starcitizen.cc \ lpi_starcraft.cc \ lpi_steam.cc \ lpi_steamfriends.cc \ @@ -183,6 +184,7 @@ libprotoident_udp_la_SOURCES = \ lpi_vpnmaster.cc \ lpi_vxworks_exploit.cc \ lpi_warthunder.cc \ + lpi_webex.cc \ lpi_wechat_udp.cc \ lpi_winmessage.cc \ lpi_wolfet.cc \ diff --git a/lib/udp/lpi_hots.cc b/lib/udp/lpi_hots.cc index d94710ae..898fd308 100644 --- a/lib/udp/lpi_hots.cc +++ b/lib/udp/lpi_hots.cc @@ -49,6 +49,8 @@ static inline bool match_hots_other(uint32_t payload, uint32_t len) { return true; if (len == 20 && MATCH(payload, ANY, ANY, 0x04, 0x00)) return true; + if (len == 20 && MATCH(payload, ANY, ANY, 0x05, 0x00)) + return true; if (len == 20 && MATCH(payload, ANY, ANY, 0x00, 0x00)) { if (MATCH(payload, 0x00, 0x00, 0x00, 0x00)) return false; diff --git a/lib/udp/lpi_starcitizen.cc b/lib/udp/lpi_starcitizen.cc new file mode 100644 index 00000000..97fe392c --- /dev/null +++ b/lib/udp/lpi_starcitizen.cc @@ -0,0 +1,78 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_sc_13(uint32_t payload, uint32_t len) { + + if (len == 13 && MATCH(payload, 0x43, 0x00, 0x00, 0x00)) + return true; + return false; + +} + +static inline bool match_sc_67(uint32_t payload, uint32_t len) { + + if (len == 67 && MATCH(payload, 0x43, 0x00, 0x00, 0x00)) + return true; + return false; + +} + +static inline bool match_starcitizen(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* Port 64090 */ + if (match_sc_13(data->payload[0], data->payload_len[0])) { + if (match_sc_13(data->payload[1], data->payload_len[1])) + return true; + if (match_sc_67(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_sc_13(data->payload[1], data->payload_len[1])) { + if (match_sc_67(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_starcitizen = { + LPI_PROTO_UDP_STARCITIZEN, + LPI_CATEGORY_GAMING, + "StarCitizen", + 39, + match_starcitizen +}; + +void register_starcitizen(LPIModuleMap *mod_map) { + register_protocol(&lpi_starcitizen, mod_map); +} + diff --git a/lib/udp/lpi_webex.cc b/lib/udp/lpi_webex.cc new file mode 100644 index 00000000..c02d456d --- /dev/null +++ b/lib/udp/lpi_webex.cc @@ -0,0 +1,78 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_webex_req(uint32_t payload, uint32_t len) { + + if (len == 8 && MATCH(payload, 0x01, 0x00, 0x00, 0x00)) + return true; + return false; +} + +static inline bool match_webex_resp(uint32_t payload, uint32_t len) { + + if (len == 8 && MATCH(payload, 0x02, 0x00, 0x00, 0x00)) + return true; + return false; +} + +static inline bool match_webex(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* Port 9000 */ + + if (data->server_port != 9000 && data->client_port != 9000) + return false; + + if (match_webex_req(data->payload[0], data->payload_len[0])) { + if (match_webex_resp(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_webex_req(data->payload[1], data->payload_len[1])) { + if (match_webex_resp(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_webex = { + LPI_PROTO_UDP_WEBEX, + LPI_CATEGORY_VOIP, + "Webex", + 100, + match_webex +}; + +void register_webex(LPIModuleMap *mod_map) { + register_protocol(&lpi_webex, mod_map); +} + diff --git a/lib/udp/udp_protocols.h b/lib/udp/udp_protocols.h index 2a838201..1e6e6cca 100644 --- a/lib/udp/udp_protocols.h +++ b/lib/udp/udp_protocols.h @@ -186,6 +186,7 @@ void register_spamfighter(LPIModuleMap *mod_map); void register_spotify_bcast(LPIModuleMap *mod_map); void register_sql_worm(LPIModuleMap *mod_map); void register_ssdp(LPIModuleMap *mod_map); +void register_starcitizen(LPIModuleMap *mod_map); void register_starcraft(LPIModuleMap *mod_map); void register_steamfriends(LPIModuleMap *mod_map); void register_steam_inhomebroadcast(LPIModuleMap *mod_map); @@ -213,6 +214,7 @@ void register_vivox(LPIModuleMap *mod_map); void register_vpnmaster(LPIModuleMap *mod_map); void register_vxworks_exploit(LPIModuleMap *mod_map); void register_warthunder(LPIModuleMap *mod_map); +void register_webex(LPIModuleMap *mod_map); void register_wechat_udp(LPIModuleMap *mod_map); void register_winmessage(LPIModuleMap *mod_map); void register_wolfet(LPIModuleMap *mod_map); From 8efcb77dedf4dc95b4b716a6f0380af4977bec00 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Thu, 25 May 2017 17:45:50 +1200 Subject: [PATCH 43/68] Add new protocol: appear.in Also added yet another HOTS pattern... --- lib/libprotoident.h | 1 + lib/proto_manager.cc | 1 + lib/tcp/Makefile.am | 1 + lib/tcp/lpi_appearin.cc | 89 +++++++++++++++++++++++++++++++++++++++++ lib/tcp/tcp_protocols.h | 1 + lib/udp/lpi_hots.cc | 2 + 6 files changed, 95 insertions(+) create mode 100644 lib/tcp/lpi_appearin.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index ec1b1146..29987c22 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -338,6 +338,7 @@ typedef enum { LPI_PROTO_FUCKCOIN, LPI_PROTO_OURWORLD, LPI_PROTO_GRAAL_ONLINE_ERA, + LPI_PROTO_APPEAR_IN, /* UDP Protocols */ LPI_PROTO_UDP, diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index e045fc2e..43d06e10 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -73,6 +73,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_airdroid(mod_map); register_akamai_tcp(mod_map); register_amp(mod_map); + register_appearin(mod_map); register_apple_push(mod_map); register_ares(mod_map); register_badbaidu(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index d7dc8df2..d7a683f9 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -6,6 +6,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_airdroid.cc \ lpi_akamai.cc \ lpi_amp.cc \ + lpi_appearin.cc \ lpi_apple_push.cc \ lpi_ares.cc \ lpi_badbaidu.cc \ diff --git a/lib/tcp/lpi_appearin.cc b/lib/tcp/lpi_appearin.cc new file mode 100644 index 00000000..5ba45a65 --- /dev/null +++ b/lib/tcp/lpi_appearin.cc @@ -0,0 +1,89 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + + +/* Signalling protocol (?) used by appear.in, a WebRTC based video + * conferencing system. + */ + +static inline bool match_appear_req(uint32_t payload, uint32_t len) { + + uint32_t hlen = ntohl(payload) & 0xffff; + + /* Usually 28 bytes */ + if (hlen == len - 20 && MATCH(payload, 0x00, 0x03, 0x00, ANY)) + return true; + return false; + +} + +static inline bool match_appear_resp(uint32_t payload, uint32_t len) { + + uint32_t hlen = ntohl(payload) & 0xffff; + + /* Usually 92 bytes */ + if (hlen == len - 20 && MATCH(payload, 0x01, 0x13, 0x00, ANY)) + return true; + return false; + +} + +static inline bool match_appearin(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (data->server_port != 443 && data->client_port != 443) + return false; + + if (match_appear_req(data->payload[0], data->payload_len[0])) { + if (match_appear_resp(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_appear_req(data->payload[1], data->payload_len[1])) { + if (match_appear_resp(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_appearin = { + LPI_PROTO_APPEAR_IN, + LPI_CATEGORY_VOIP, + "Appear.in", + 120, + match_appearin +}; + +void register_appearin(LPIModuleMap *mod_map) { + register_protocol(&lpi_appearin, mod_map); +} + diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index 33dda8e8..72c38be1 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -34,6 +34,7 @@ void register_afp(LPIModuleMap *mod_map); void register_airdroid(LPIModuleMap *mod_map); void register_akamai_tcp(LPIModuleMap *mod_map); void register_amp(LPIModuleMap *mod_map); +void register_appearin(LPIModuleMap *mod_map); void register_apple_push(LPIModuleMap *mod_map); void register_ares(LPIModuleMap *mod_map); void register_badbaidu(LPIModuleMap *mod_map); diff --git a/lib/udp/lpi_hots.cc b/lib/udp/lpi_hots.cc index 898fd308..8ec06990 100644 --- a/lib/udp/lpi_hots.cc +++ b/lib/udp/lpi_hots.cc @@ -51,6 +51,8 @@ static inline bool match_hots_other(uint32_t payload, uint32_t len) { return true; if (len == 20 && MATCH(payload, ANY, ANY, 0x05, 0x00)) return true; + if (len == 20 && MATCH(payload, ANY, ANY, 0x06, 0x00)) + return true; if (len == 20 && MATCH(payload, ANY, ANY, 0x00, 0x00)) { if (MATCH(payload, 0x00, 0x00, 0x00, 0x00)) return false; From 90bedfd5d35164617414d19e6634bc629b83d088 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Tue, 30 May 2017 15:51:07 +1200 Subject: [PATCH 44/68] Added two new game protocols: Vainglory and QQ Weiqi Also added the next HOTS pattern :) --- lib/libprotoident.h | 2 ++ lib/proto_manager.cc | 2 ++ lib/tcp/Makefile.am | 2 ++ lib/tcp/lpi_vainglory.cc | 77 ++++++++++++++++++++++++++++++++++++++++ lib/tcp/lpi_weiqi.cc | 76 +++++++++++++++++++++++++++++++++++++++ lib/tcp/tcp_protocols.h | 2 ++ lib/udp/lpi_hots.cc | 2 ++ 7 files changed, 163 insertions(+) create mode 100644 lib/tcp/lpi_vainglory.cc create mode 100644 lib/tcp/lpi_weiqi.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 29987c22..68f7a751 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -339,6 +339,8 @@ typedef enum { LPI_PROTO_OURWORLD, LPI_PROTO_GRAAL_ONLINE_ERA, LPI_PROTO_APPEAR_IN, + LPI_PROTO_VAINGLORY, + LPI_PROTO_WEIQI, /* UDP Protocols */ LPI_PROTO_UDP, diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index 43d06e10..c1fcb4fe 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -262,6 +262,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_trojan_zeroaccess(mod_map); register_twitcasting(mod_map); register_twitch_irc(mod_map); + register_vainglory(mod_map); register_viber(mod_map); register_vodlocker(mod_map); register_warcraft3(mod_map); @@ -269,6 +270,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_weblogic(mod_map); register_wechat(mod_map); register_weibo(mod_map); + register_weiqi(mod_map); register_whatsapp(mod_map); register_whois(mod_map); register_winmx(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index d7a683f9..c0b1c04b 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -195,6 +195,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_trojan_zeroaccess.cc \ lpi_twitcasting.cc \ lpi_twitch_irc.cc \ + lpi_vainglory.cc \ lpi_viber.cc \ lpi_vodlocker.cc \ lpi_warcraft3.cc \ @@ -202,6 +203,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_weblogic.cc \ lpi_wechat.cc \ lpi_weibo.cc \ + lpi_weiqi.cc \ lpi_whatsapp.cc \ lpi_whois.cc \ lpi_winmx.cc \ diff --git a/lib/tcp/lpi_vainglory.cc b/lib/tcp/lpi_vainglory.cc new file mode 100644 index 00000000..285b0b89 --- /dev/null +++ b/lib/tcp/lpi_vainglory.cc @@ -0,0 +1,77 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* Vainglory -- MOBA for touch screens */ + +static inline bool match_vg_req(uint32_t payload, uint32_t len) { + + if (MATCH(payload, 0x00, 0x86, 0x00, 0x06) && len == 136) + return true; + return false; + +} + +static inline bool match_vg_resp(uint32_t payload, uint32_t len) { + + if (MATCH(payload, 0x00, 0x03, 0x00, 0x07) && len == 5) + return true; + return false; + +} + +static inline bool match_vainglory(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_vg_req(data->payload[0], data->payload_len[0])) { + if (match_vg_resp(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_vg_req(data->payload[1], data->payload_len[1])) { + if (match_vg_resp(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_vainglory = { + LPI_PROTO_VAINGLORY, + LPI_CATEGORY_GAMING, + "Vainglory", + 5, + match_vainglory +}; + +void register_vainglory(LPIModuleMap *mod_map) { + register_protocol(&lpi_vainglory, mod_map); +} + diff --git a/lib/tcp/lpi_weiqi.cc b/lib/tcp/lpi_weiqi.cc new file mode 100644 index 00000000..f272d1cb --- /dev/null +++ b/lib/tcp/lpi_weiqi.cc @@ -0,0 +1,76 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* AKA Fox Go -- Multiplayer Go client included with QQ Games */ + +static inline bool match_weiqi_tgw(uint32_t payload, uint32_t len) { + if (len == 43 && MATCH(payload, 't', 'g', 'w', '_')) + return true; + return false; +} + +static inline bool match_weiqi_other(uint32_t payload, uint32_t len) { + + if (MATCH(payload, 0x00, 0x0e, 0x0f, 0xa5)) + return true; + return false; +} + +static inline bool match_weiqi(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* Port 8000 */ + + if (match_weiqi_tgw(data->payload[0], data->payload_len[0])) { + if (match_weiqi_other(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_weiqi_tgw(data->payload[1], data->payload_len[1])) { + if (match_weiqi_other(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_weiqi = { + LPI_PROTO_WEIQI, + LPI_CATEGORY_GAMING, + "WeiqiQQ", + 20, + match_weiqi +}; + +void register_weiqi(LPIModuleMap *mod_map) { + register_protocol(&lpi_weiqi, mod_map); +} + diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index 72c38be1..6d53bdc2 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -223,6 +223,7 @@ void register_trojan_win32_generic_sb(LPIModuleMap *mod_map); void register_trojan_zeroaccess(LPIModuleMap *mod_map); void register_twitcasting(LPIModuleMap *mod_map); void register_twitch_irc(LPIModuleMap *mod_map); +void register_vainglory(LPIModuleMap *mod_map); void register_viber(LPIModuleMap *mod_map); void register_vodlocker(LPIModuleMap *mod_map); void register_warcraft3(LPIModuleMap *mod_map); @@ -230,6 +231,7 @@ void register_web_junk(LPIModuleMap *mod_map); void register_weblogic(LPIModuleMap *mod_map); void register_wechat(LPIModuleMap *mod_map); void register_weibo(LPIModuleMap *mod_map); +void register_weiqi(LPIModuleMap *mod_map); void register_whatsapp(LPIModuleMap *mod_map); void register_whois(LPIModuleMap *mod_map); void register_winmx(LPIModuleMap *mod_map); diff --git a/lib/udp/lpi_hots.cc b/lib/udp/lpi_hots.cc index 8ec06990..1625436b 100644 --- a/lib/udp/lpi_hots.cc +++ b/lib/udp/lpi_hots.cc @@ -53,6 +53,8 @@ static inline bool match_hots_other(uint32_t payload, uint32_t len) { return true; if (len == 20 && MATCH(payload, ANY, ANY, 0x06, 0x00)) return true; + if (len == 20 && MATCH(payload, ANY, ANY, 0x07, 0x00)) + return true; if (len == 20 && MATCH(payload, ANY, ANY, 0x00, 0x00)) { if (MATCH(payload, 0x00, 0x00, 0x00, 0x00)) return false; From ffc4c91ed47e2262bb0f8a11379b1c6ab9e8936a Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 2 Jun 2017 09:17:26 +1200 Subject: [PATCH 45/68] Two more new protocols: 4D and Halo Online --- lib/libprotoident.h | 2 + lib/proto_manager.cc | 2 + lib/tcp/Makefile.am | 1 + lib/tcp/lpi_4d.cc | 77 ++++++++++++++++++++++++++++++++++++++ lib/tcp/tcp_protocols.h | 1 + lib/udp/Makefile.am | 1 + lib/udp/lpi_haloonline.cc | 78 +++++++++++++++++++++++++++++++++++++++ lib/udp/udp_protocols.h | 1 + 8 files changed, 163 insertions(+) create mode 100644 lib/tcp/lpi_4d.cc create mode 100644 lib/udp/lpi_haloonline.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 68f7a751..341d967c 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -341,6 +341,7 @@ typedef enum { LPI_PROTO_APPEAR_IN, LPI_PROTO_VAINGLORY, LPI_PROTO_WEIQI, + LPI_PROTO_4D, /* UDP Protocols */ LPI_PROTO_UDP, @@ -531,6 +532,7 @@ typedef enum { LPI_PROTO_UDP_PS4_REMOTEPLAY, LPI_PROTO_UDP_STARCITIZEN, LPI_PROTO_UDP_WEBEX, + LPI_PROTO_UDP_HALO_ONLINE, /* Patterns that we can match, but do not know the protocol */ LPI_PROTO_REJECTION, /* All responses are 0x02 */ diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index c1fcb4fe..63a2a3b6 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -68,6 +68,7 @@ void free_protocols(LPIModuleMap *mod_map) { int register_tcp_protocols(LPIModuleMap *mod_map) { register_360safeguard(mod_map); + register_4d(mod_map); register_acestream(mod_map); register_afp(mod_map); register_airdroid(mod_map); @@ -363,6 +364,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_gsm(mod_map); register_h1z1(mod_map); register_halflife(mod_map); + register_haloonline(mod_map); register_hamachi_udp(mod_map); register_heroes_generals(mod_map); register_hots(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index c0b1c04b..ef74d026 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -1,6 +1,7 @@ noinst_LTLIBRARIES = libprotoident_tcp.la libprotoident_tcp_la_SOURCES = \ lpi_360safeguard.cc \ + lpi_4d.cc \ lpi_acestream.cc \ lpi_afp.cc \ lpi_airdroid.cc \ diff --git a/lib/tcp/lpi_4d.cc b/lib/tcp/lpi_4d.cc new file mode 100644 index 00000000..030a745e --- /dev/null +++ b/lib/tcp/lpi_4d.cc @@ -0,0 +1,77 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_4d_5f(uint32_t payload, uint32_t len) { + + if (len == 4 && MATCH(payload, 0x5d, 0x11, 0x00, 0x00)) + return true; + return false; + +} + +static inline bool match_4d_other(uint32_t payload, uint32_t len) { + + if (len == 4 && MATCH(payload, 0x60, 0x11, 0x00, 0x00)) + return true; + if (len == 4 && MATCH(payload, 0x61, 0x11, 0x00, 0x00)) + return true; + return false; + +} + +static inline bool match_4d(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_4d_5f(data->payload[0], data->payload_len[0])) { + if (match_4d_other(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_4d_5f(data->payload[1], data->payload_len[1])) { + if (match_4d_other(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_4d = { + LPI_PROTO_4D, + LPI_CATEGORY_DATABASES, + "4D", + 75, + match_4d +}; + +void register_4d(LPIModuleMap *mod_map) { + register_protocol(&lpi_4d, mod_map); +} + diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index 6d53bdc2..3348ab44 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -29,6 +29,7 @@ #include "proto_manager.h" void register_360safeguard(LPIModuleMap *mod_map); +void register_4d(LPIModuleMap *mod_map); void register_acestream(LPIModuleMap *mod_map); void register_afp(LPIModuleMap *mod_map); void register_airdroid(LPIModuleMap *mod_map); diff --git a/lib/udp/Makefile.am b/lib/udp/Makefile.am index 5850548c..84ce90bc 100644 --- a/lib/udp/Makefile.am +++ b/lib/udp/Makefile.am @@ -65,6 +65,7 @@ libprotoident_udp_la_SOURCES = \ lpi_gsm.cc \ lpi_h1z1.cc \ lpi_halflife.cc \ + lpi_haloonline.cc \ lpi_hamachi.cc \ lpi_heroes_generals.cc \ lpi_hots.cc \ diff --git a/lib/udp/lpi_haloonline.cc b/lib/udp/lpi_haloonline.cc new file mode 100644 index 00000000..12553bf8 --- /dev/null +++ b/lib/udp/lpi_haloonline.cc @@ -0,0 +1,78 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* Halo Online -- multiplayer FPS that has already been shut down but is + * somehow still being played by FPS "aficionados". + */ + +static inline bool match_bla_17(uint32_t payload, uint32_t len) { + if (len == 17 && MATCH(payload, 0x01, 'b', 'l', 'a')) + return true; + return false; + +} + +static inline bool match_bla_86(uint32_t payload, uint32_t len) { + if (len == 86 && MATCH(payload, 0x01, 'b', 'l', 'a')) + return true; + return false; + +} + +static inline bool match_haloonline(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* Port 11774 */ + if (match_bla_17(data->payload[0], data->payload_len[0])) { + if (match_bla_86(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_bla_17(data->payload[1], data->payload_len[1])) { + if (match_bla_86(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_haloonline = { + LPI_PROTO_UDP_HALO_ONLINE, + LPI_CATEGORY_GAMING, + "HaloOnline", + 5, + match_haloonline +}; + +void register_haloonline(LPIModuleMap *mod_map) { + register_protocol(&lpi_haloonline, mod_map); +} + diff --git a/lib/udp/udp_protocols.h b/lib/udp/udp_protocols.h index 1e6e6cca..de0ff50a 100644 --- a/lib/udp/udp_protocols.h +++ b/lib/udp/udp_protocols.h @@ -95,6 +95,7 @@ void register_gprs_tunnel(LPIModuleMap *mod_map); void register_gsm(LPIModuleMap *mod_map); void register_h1z1(LPIModuleMap *mod_map); void register_halflife(LPIModuleMap *mod_map); +void register_haloonline(LPIModuleMap *mod_map); void register_hamachi_udp(LPIModuleMap *mod_map); void register_heroes_generals(LPIModuleMap *mod_map); void register_hots(LPIModuleMap *mod_map); From 0e8808bac86d87d61ae4a45443bdc88e90e574cb Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 2 Jun 2017 11:38:21 +1200 Subject: [PATCH 46/68] Add new protocol: GoToMeeting Also added some new patterns to Fortinet --- lib/libprotoident.h | 1 + lib/proto_manager.cc | 1 + lib/udp/Makefile.am | 1 + lib/udp/lpi_fortinet.cc | 4 ++ lib/udp/lpi_gotomeeting.cc | 95 ++++++++++++++++++++++++++++++++++++++ lib/udp/udp_protocols.h | 1 + 6 files changed, 103 insertions(+) create mode 100644 lib/udp/lpi_gotomeeting.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 341d967c..cde1bef6 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -533,6 +533,7 @@ typedef enum { LPI_PROTO_UDP_STARCITIZEN, LPI_PROTO_UDP_WEBEX, LPI_PROTO_UDP_HALO_ONLINE, + LPI_PROTO_UDP_GOTOMEETING, /* Patterns that we can match, but do not know the protocol */ LPI_PROTO_REJECTION, /* All responses are 0x02 */ diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index 63a2a3b6..d5bd0c32 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -360,6 +360,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_gnutella_udp(mod_map); register_gnutella2_udp(mod_map); register_gnutella_weak(mod_map); + register_gotomeeting(mod_map); register_gprs_tunnel(mod_map); register_gsm(mod_map); register_h1z1(mod_map); diff --git a/lib/udp/Makefile.am b/lib/udp/Makefile.am index 84ce90bc..995afaea 100644 --- a/lib/udp/Makefile.am +++ b/lib/udp/Makefile.am @@ -61,6 +61,7 @@ libprotoident_udp_la_SOURCES = \ lpi_gnutella.cc \ lpi_gnutella2.cc \ lpi_gnutella_weak.cc \ + lpi_gotomeeting.cc \ lpi_gprs_tunnel.cc \ lpi_gsm.cc \ lpi_h1z1.cc \ diff --git a/lib/udp/lpi_fortinet.cc b/lib/udp/lpi_fortinet.cc index d2a79f62..55dd6809 100644 --- a/lib/udp/lpi_fortinet.cc +++ b/lib/udp/lpi_fortinet.cc @@ -60,6 +60,8 @@ static inline bool match_fortinet_req(uint32_t payload, uint32_t len) { return true; if (MATCHSTR(payload, "ikvk")) return true; + if (MATCHSTR(payload, "ikvo")) + return true; return false; @@ -77,6 +79,8 @@ static inline bool match_fortinet_resp(uint32_t payload, uint32_t len) { return true; if (len == 32 && MATCHSTR(payload, "khwK")) return true; + if (len == 64 && MATCHSTR(payload, "kkw+")) + return true; return false; } diff --git a/lib/udp/lpi_gotomeeting.cc b/lib/udp/lpi_gotomeeting.cc new file mode 100644 index 00000000..a1821290 --- /dev/null +++ b/lib/udp/lpi_gotomeeting.cc @@ -0,0 +1,95 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_gtm_ping(uint32_t payload, uint32_t len) { + + if (len == 16 && MATCH(payload, 'P', 'I', 'N', 'G')) + return true; + return false; +} + +static inline bool match_gtm_pong(uint32_t payload, uint32_t len) { + + if (len == 16 && MATCH(payload, 'P', 'O', 'N', 'G')) + return true; + return false; +} + +static inline bool match_gtm_webcam(uint32_t ploada, uint32_t ploadb) { + + /* Bytes 2,3,4 match, but be careful not to match stuff like + * 0x000000 */ + + if (MATCH(ploada, ANY, 0x00, 0x00, 0x00)) + return false; + + if ((ploada & 0xffffff00) == (ploadb & 0xffffff00)) + return true; + return false; + +} + +static inline bool match_gotomeeting(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* VOIP tends to be on port 8200 */ + + if (match_gtm_ping(data->payload[0], data->payload_len[0])) { + if (match_gtm_pong(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_gtm_ping(data->payload[1], data->payload_len[1])) { + if (match_gtm_pong(data->payload[0], data->payload_len[0])) + return true; + } + + /* Webcam goes over port 1853 */ + if (match_gtm_webcam(data->payload[0], data->payload[1])) { + if (data->server_port == 1853 || data->client_port == 1853) + return true; + } + + return false; +} + +static lpi_module_t lpi_gotomeeting = { + LPI_PROTO_UDP_GOTOMEETING, + LPI_CATEGORY_VOIP, + "GoToMeeting", + 149, + match_gotomeeting +}; + +void register_gotomeeting(LPIModuleMap *mod_map) { + register_protocol(&lpi_gotomeeting, mod_map); +} + diff --git a/lib/udp/udp_protocols.h b/lib/udp/udp_protocols.h index de0ff50a..991012df 100644 --- a/lib/udp/udp_protocols.h +++ b/lib/udp/udp_protocols.h @@ -91,6 +91,7 @@ void register_gearsofwar(LPIModuleMap *mod_map); void register_gnutella_udp(LPIModuleMap *mod_map); void register_gnutella2_udp(LPIModuleMap *mod_map); void register_gnutella_weak(LPIModuleMap *mod_map); +void register_gotomeeting(LPIModuleMap *mod_map); void register_gprs_tunnel(LPIModuleMap *mod_map); void register_gsm(LPIModuleMap *mod_map); void register_h1z1(LPIModuleMap *mod_map); From df409186c5ca273b414ea57f4a403a05bfa5e6b6 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 2 Jun 2017 11:47:49 +1200 Subject: [PATCH 47/68] Fix missing '&' in lpi_vivox module :( --- lib/udp/lpi_vivox.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/udp/lpi_vivox.cc b/lib/udp/lpi_vivox.cc index 57be1b26..be7aeae5 100644 --- a/lib/udp/lpi_vivox.cc +++ b/lib/udp/lpi_vivox.cc @@ -32,7 +32,7 @@ static inline bool match_vivox_request(uint32_t payload, uint32_t len) { - if (MATCH(payload, 0x80, 0x7f, 0x00, 0x00) & len == 112) + if (MATCH(payload, 0x80, 0x7f, 0x00, 0x00) && len == 112) return true; return false; @@ -40,7 +40,7 @@ static inline bool match_vivox_request(uint32_t payload, uint32_t len) { static inline bool match_vivox_reply(uint32_t payload, uint32_t len) { - if (MATCH(payload, 0x80, 0x7f, 0x00, 0x00) & len == 99) + if (MATCH(payload, 0x80, 0x7f, 0x00, 0x00) && len == 99) return true; return false; @@ -51,7 +51,7 @@ static inline bool match_vivox_stun(uint32_t payload, uint32_t len) { /* Vivox mixes a whole bunch of RTP(?) and STUN together * when it starts up, so sometimes the STUN replies arrive * first. */ - if (MATCH(payload, 0x01, 0x01, 0x00, 0x70) & len == 132) + if (MATCH(payload, 0x01, 0x01, 0x00, 0x70) && len == 132) return true; return false; From 8ea9f0371fc9af5c07bcf8ca7caae8441ebd507e Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 2 Jun 2017 11:55:50 +1200 Subject: [PATCH 48/68] Added some new patterns for DOTA2 Not sure if they've changed the protocol recently? --- lib/udp/lpi_dota2.cc | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/lib/udp/lpi_dota2.cc b/lib/udp/lpi_dota2.cc index 715b700f..537f83ca 100644 --- a/lib/udp/lpi_dota2.cc +++ b/lib/udp/lpi_dota2.cc @@ -69,6 +69,22 @@ static inline bool match_dota2_0212(uint32_t payload, uint32_t len) { return false; } +static inline bool match_dota2_512(uint32_t payload, uint32_t len) { + + if (len == 512 && MATCH(payload, 0x20, 0x0e, 0x00, 0x0d)) + return true; + return false; + +} + +static inline bool match_dota2_24(uint32_t payload, uint32_t len) { + + if (len == 24 && MATCH(payload, 0x21, 0x0d, ANY, ANY)) + return true; + return false; + +} + static inline bool match_dota2(lpi_data_t *data, lpi_module_t *mod UNUSED) { @@ -92,6 +108,16 @@ static inline bool match_dota2(lpi_data_t *data, lpi_module_t *mod UNUSED) { return true; } + if (match_dota2_512(data->payload[0], data->payload_len[0])) { + if (match_dota2_24(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_dota2_512(data->payload[1], data->payload_len[1])) { + if (match_dota2_24(data->payload[0], data->payload_len[0])) + return true; + } + return false; } From 5127dd3db2380997d17a07adb39774269d8d3aa5 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Tue, 6 Jun 2017 17:47:32 +1200 Subject: [PATCH 49/68] Added another new protocol: Tanki X Also added new patterns to: * Black Desert Online * Heroes of the Storm * Warthunder --- lib/libprotoident.h | 1 + lib/proto_manager.cc | 1 + lib/tcp/Makefile.am | 1 + lib/tcp/lpi_blackdesert.cc | 2 ++ lib/tcp/lpi_tankix.cc | 68 ++++++++++++++++++++++++++++++++++++++ lib/tcp/tcp_protocols.h | 1 + lib/udp/lpi_hots.cc | 2 ++ lib/udp/lpi_warthunder.cc | 2 ++ 8 files changed, 78 insertions(+) create mode 100644 lib/tcp/lpi_tankix.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index cde1bef6..670da4d5 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -342,6 +342,7 @@ typedef enum { LPI_PROTO_VAINGLORY, LPI_PROTO_WEIQI, LPI_PROTO_4D, + LPI_PROTO_TANKIX, /* UDP Protocols */ LPI_PROTO_UDP, diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index d5bd0c32..ca1ccd04 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -242,6 +242,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_stun_tcp(mod_map); register_supl(mod_map); register_svn(mod_map); + register_tankix(mod_map); register_taobao(mod_map); register_tds(mod_map); register_teamviewer(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index ef74d026..a74e1588 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -175,6 +175,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_stun.cc \ lpi_supl.cc \ lpi_svn.cc \ + lpi_tankix.cc \ lpi_taobao.cc \ lpi_tds.cc \ lpi_teamviewer.cc \ diff --git a/lib/tcp/lpi_blackdesert.cc b/lib/tcp/lpi_blackdesert.cc index 1c59d783..99a809d1 100644 --- a/lib/tcp/lpi_blackdesert.cc +++ b/lib/tcp/lpi_blackdesert.cc @@ -47,6 +47,8 @@ static inline bool match_bdo_reply(uint32_t payload, uint32_t len) { if (len == 112 && MATCH(payload, 0x70, 0x00, 0x01, ANY)) return true; + if (len == 113 && MATCH(payload, 0x71, 0x00, 0x01, ANY)) + return true; return false; diff --git a/lib/tcp/lpi_tankix.cc b/lib/tcp/lpi_tankix.cc new file mode 100644 index 00000000..c85b32e4 --- /dev/null +++ b/lib/tcp/lpi_tankix.cc @@ -0,0 +1,68 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_tankix_19(uint32_t payload, uint32_t len) { + + if (MATCHSTR(payload, "\xff\x00\x00\x00") && len == 19) + return true; + return false; +} + + +static inline bool match_tankix(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* Port 5050 */ + if (match_tankix_19(data->payload[0], data->payload_len[0])) { + if (MATCHSTR(data->payload[1], "\xff\x00\x00\x00")) + return true; + } + + if (match_tankix_19(data->payload[1], data->payload_len[1])) { + if (MATCHSTR(data->payload[0], "\xff\x00\x00\x00")) + return true; + } + + return false; +} + +static lpi_module_t lpi_tankix = { + LPI_PROTO_TANKIX, + LPI_CATEGORY_GAMING, + "TankiX", + 52, + match_tankix +}; + +void register_tankix(LPIModuleMap *mod_map) { + register_protocol(&lpi_tankix, mod_map); +} + diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index 3348ab44..c9ea06cb 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -203,6 +203,7 @@ void register_steam(LPIModuleMap *mod_map); void register_stun_tcp(LPIModuleMap *mod_map); void register_supl(LPIModuleMap *mod_map); void register_svn(LPIModuleMap *mod_map); +void register_tankix(LPIModuleMap *mod_map); void register_taobao(LPIModuleMap *mod_map); void register_tds(LPIModuleMap *mod_map); void register_teamviewer(LPIModuleMap *mod_map); diff --git a/lib/udp/lpi_hots.cc b/lib/udp/lpi_hots.cc index 1625436b..3544c6f0 100644 --- a/lib/udp/lpi_hots.cc +++ b/lib/udp/lpi_hots.cc @@ -55,6 +55,8 @@ static inline bool match_hots_other(uint32_t payload, uint32_t len) { return true; if (len == 20 && MATCH(payload, ANY, ANY, 0x07, 0x00)) return true; + if (len == 20 && MATCH(payload, ANY, ANY, 0x08, 0x00)) + return true; if (len == 20 && MATCH(payload, ANY, ANY, 0x00, 0x00)) { if (MATCH(payload, 0x00, 0x00, 0x00, 0x00)) return false; diff --git a/lib/udp/lpi_warthunder.cc b/lib/udp/lpi_warthunder.cc index 391dad0f..60f080a3 100644 --- a/lib/udp/lpi_warthunder.cc +++ b/lib/udp/lpi_warthunder.cc @@ -38,6 +38,8 @@ static inline bool match_warthunder_req(uint32_t payload, uint32_t len) { return true; if (len == 52 && MATCH(payload, 0xcf, 0xff, 0x00, 0x0b)) return true; + if (len == 52 && MATCH(payload, 0xcf, 0xff, 0x00, 0x05)) + return true; if (len == 52 && MATCH(payload, 0xcf, 0xff, 0x00, 0x14)) return true; return false; From 98ece6f0baaa861daf56ebc398583a4d4e3b4804 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Wed, 7 Jun 2017 17:52:58 +1200 Subject: [PATCH 50/68] Added new protocol: IPSharkk --- lib/libprotoident.h | 1 + lib/proto_manager.cc | 1 + lib/tcp/Makefile.am | 1 + lib/tcp/lpi_ipsharkk.cc | 76 +++++++++++++++++++++++++++++++++++++++++ lib/tcp/tcp_protocols.h | 1 + 5 files changed, 80 insertions(+) create mode 100644 lib/tcp/lpi_ipsharkk.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 670da4d5..a65f16a8 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -343,6 +343,7 @@ typedef enum { LPI_PROTO_WEIQI, LPI_PROTO_4D, LPI_PROTO_TANKIX, + LPI_PROTO_IPSHARKK, /* UDP Protocols */ LPI_PROTO_UDP, diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index ca1ccd04..7d20c8e3 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -147,6 +147,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_invalid_pop(mod_map); register_invalid_smtp(mod_map); register_ipop(mod_map); + register_ipsharkk(mod_map); register_irc(mod_map); register_java(mod_map); register_jedi(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index a74e1588..c090ffb3 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -80,6 +80,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_invalid_pop.cc \ lpi_invalid_smtp.cc \ lpi_ipop.cc \ + lpi_ipsharkk.cc \ lpi_irc.cc \ lpi_java.cc \ lpi_jedi.cc \ diff --git a/lib/tcp/lpi_ipsharkk.cc b/lib/tcp/lpi_ipsharkk.cc new file mode 100644 index 00000000..6b410a4b --- /dev/null +++ b/lib/tcp/lpi_ipsharkk.cc @@ -0,0 +1,76 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* IPSharkk P2P Proxy */ + +static inline bool match_ipsharkk_ssl(uint32_t payload) { + + if (MATCH(payload, 0x16, 0x03, 0x03, 0x00)) + return true; + return false; + +} + +static inline bool match_ipsharkk_4f(uint32_t payload) { + + if (MATCH(payload, 0x4f, 0x1b, 0x4d, 0x07)) + return true; + return false; +} + +static inline bool match_ipsharkk(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_ipsharkk_ssl(data->payload[0])) { + if (match_ipsharkk_4f(data->payload[1])) + return true; + } + + if (match_ipsharkk_ssl(data->payload[1])) { + if (match_ipsharkk_4f(data->payload[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_ipsharkk = { + LPI_PROTO_IPSHARKK, + LPI_CATEGORY_TUNNELLING, + "IPSharkk", + 15, + match_ipsharkk +}; + +void register_ipsharkk(LPIModuleMap *mod_map) { + register_protocol(&lpi_ipsharkk, mod_map); +} + diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index c9ea06cb..2c89b52b 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -108,6 +108,7 @@ void register_invalid_http(LPIModuleMap *mod_map); void register_invalid_pop(LPIModuleMap *mod_map); void register_invalid_smtp(LPIModuleMap *mod_map); void register_ipop(LPIModuleMap *mod_map); +void register_ipsharkk(LPIModuleMap *mod_map); void register_irc(LPIModuleMap *mod_map); void register_java(LPIModuleMap *mod_map); void register_jedi(LPIModuleMap *mod_map); From 4b999db4128f867dff09a13a060827a792dc9255 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Thu, 8 Jun 2017 11:36:30 +1200 Subject: [PATCH 51/68] Added 2 more protocols: Umeye and Crossout --- lib/libprotoident.h | 2 ++ lib/proto_manager.cc | 2 ++ lib/udp/Makefile.am | 2 ++ lib/udp/lpi_crossout.cc | 75 +++++++++++++++++++++++++++++++++++++++++ lib/udp/lpi_umeye.cc | 67 ++++++++++++++++++++++++++++++++++++ lib/udp/udp_protocols.h | 2 ++ 6 files changed, 150 insertions(+) create mode 100644 lib/udp/lpi_crossout.cc create mode 100644 lib/udp/lpi_umeye.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index a65f16a8..f391a3e6 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -536,6 +536,8 @@ typedef enum { LPI_PROTO_UDP_WEBEX, LPI_PROTO_UDP_HALO_ONLINE, LPI_PROTO_UDP_GOTOMEETING, + LPI_PROTO_UDP_CROSSOUT, + LPI_PROTO_UDP_UMEYE, /* Patterns that we can match, but do not know the protocol */ LPI_PROTO_REJECTION, /* All responses are 0x02 */ diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index 7d20c8e3..2f7ca0c7 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -328,6 +328,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_cirn(mod_map); register_cisco_ipsec(mod_map); register_cisco_sslvpn(mod_map); + register_crossout(mod_map); register_csgo(mod_map); register_db2(mod_map); register_dcc_udp(mod_map); @@ -480,6 +481,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_tvants(mod_map); register_ubisoft_games(mod_map); register_udp_no_payload(mod_map); + register_umeye(mod_map); register_unreal(mod_map); register_ventrilo(mod_map); register_viber_udp(mod_map); diff --git a/lib/udp/Makefile.am b/lib/udp/Makefile.am index 995afaea..54cefaf4 100644 --- a/lib/udp/Makefile.am +++ b/lib/udp/Makefile.am @@ -27,6 +27,7 @@ libprotoident_udp_la_SOURCES = \ lpi_cisco_ipsec.cc \ lpi_cisco_sslvpn.cc \ lpi_cod.cc \ + lpi_crossout.cc \ lpi_csgo.cc \ lpi_db2.cc \ lpi_dcc.cc \ @@ -179,6 +180,7 @@ libprotoident_udp_la_SOURCES = \ lpi_turbovpn.cc \ lpi_tvants.cc \ lpi_ubisoft_games.cc \ + lpi_umeye.cc \ lpi_unreal.cc \ lpi_ventrilo.cc \ lpi_viber.cc \ diff --git a/lib/udp/lpi_crossout.cc b/lib/udp/lpi_crossout.cc new file mode 100644 index 00000000..da8a268e --- /dev/null +++ b/lib/udp/lpi_crossout.cc @@ -0,0 +1,75 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_co_05(uint32_t payload, uint32_t len) { + + if (len == 1322 && MATCHSTR(payload, "\x05\x00\xff\xff")) + return true; + return false; + +} + +static inline bool match_co_06(uint32_t payload, uint32_t len) { + + if (len == 1322 && MATCHSTR(payload, "\x06\x00\xff\xff")) + return true; + return false; + +} + +static inline bool match_crossout(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_co_05(data->payload[0], data->payload_len[0])) { + if (match_co_06(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_co_05(data->payload[1], data->payload_len[1])) { + if (match_co_06(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_crossout = { + LPI_PROTO_UDP_CROSSOUT, + LPI_CATEGORY_GAMING, + "Crossout", + 11, + match_crossout +}; + +void register_crossout(LPIModuleMap *mod_map) { + register_protocol(&lpi_crossout, mod_map); +} + diff --git a/lib/udp/lpi_umeye.cc b/lib/udp/lpi_umeye.cc new file mode 100644 index 00000000..a020cc75 --- /dev/null +++ b/lib/udp/lpi_umeye.cc @@ -0,0 +1,67 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* Umeye itself seems to be a cloud platform for streaming, but it seems + * to be most commonly used in concert with home surveillance cameras + * to allow users to check their cameras remotely. + */ + +static inline bool match_umeye_shake(uint32_t payload, uint32_t len) { + + if (len == 164 && MATCHSTR(payload, "\xc1\xef\xab\xff")) + return true; + return false; + +} + +static inline bool match_umeye(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_umeye_shake(data->payload[0], data->payload_len[0])) { + if (match_umeye_shake(data->payload[1], data->payload_len[1])) + return true; + } + + return false; +} + +static lpi_module_t lpi_umeye = { + LPI_PROTO_UDP_UMEYE, + LPI_CATEGORY_STREAMING, + "Umeye", + 101, + match_umeye +}; + +void register_umeye(LPIModuleMap *mod_map) { + register_protocol(&lpi_umeye, mod_map); +} + diff --git a/lib/udp/udp_protocols.h b/lib/udp/udp_protocols.h index 991012df..7464fa39 100644 --- a/lib/udp/udp_protocols.h +++ b/lib/udp/udp_protocols.h @@ -57,6 +57,7 @@ void register_chivalry(LPIModuleMap *mod_map); void register_cirn(LPIModuleMap *mod_map); void register_cisco_ipsec(LPIModuleMap *mod_map); void register_cisco_sslvpn(LPIModuleMap *mod_map); +void register_crossout(LPIModuleMap *mod_map); void register_csgo(LPIModuleMap *mod_map); void register_db2(LPIModuleMap *mod_map); void register_dcc_udp(LPIModuleMap *mod_map); @@ -209,6 +210,7 @@ void register_turbovpn(LPIModuleMap *mod_map); void register_tvants(LPIModuleMap *mod_map); void register_ubisoft_games(LPIModuleMap *mod_map); void register_udp_no_payload(LPIModuleMap *mod_map); +void register_umeye(LPIModuleMap *mod_map); void register_unreal(LPIModuleMap *mod_map); void register_ventrilo(LPIModuleMap *mod_map); void register_viber_udp(LPIModuleMap *mod_map); From bfd06ee0b0250232540ff9b85560db1d038a8425 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 9 Jun 2017 11:32:54 +1200 Subject: [PATCH 52/68] Added another new protocol: .NET Message Forwarding Also added a new pattern for Zoom (possibly an older version) --- lib/libprotoident.h | 1 + lib/proto_manager.cc | 1 + lib/tcp/Makefile.am | 1 + lib/tcp/lpi_netmfp.cc | 84 +++++++++++++++++++++++++++++++++++++++++ lib/tcp/tcp_protocols.h | 1 + lib/udp/lpi_zoom.cc | 2 + 6 files changed, 90 insertions(+) create mode 100644 lib/tcp/lpi_netmfp.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index f391a3e6..76796d61 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -344,6 +344,7 @@ typedef enum { LPI_PROTO_4D, LPI_PROTO_TANKIX, LPI_PROTO_IPSHARKK, + LPI_PROTO_NET_MFP, /* UDP Protocols */ LPI_PROTO_UDP, diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index 2f7ca0c7..25d54f02 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -187,6 +187,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_ndt_tput(mod_map); register_netbios(mod_map); register_netcat_cctv(mod_map); + register_netmfp(mod_map); register_nntp(mod_map); register_nntps(mod_map); register_norton_backup(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index c090ffb3..018b2870 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -120,6 +120,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_ndt_tput.cc \ lpi_netbios.cc \ lpi_netcat_cctv.cc \ + lpi_netmfp.cc \ lpi_nntp.cc \ lpi_nntps.cc \ lpi_nopayload.cc \ diff --git a/lib/tcp/lpi_netmfp.cc b/lib/tcp/lpi_netmfp.cc new file mode 100644 index 00000000..28daef26 --- /dev/null +++ b/lib/tcp/lpi_netmfp.cc @@ -0,0 +1,84 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* .NET Message Framing Protocol */ + +static inline bool match_version_record(uint32_t payload, uint32_t len) { + + /* Length will probably vary */ + /* Version, mode and via records are often included in the same + * packet */ + + if (MATCH(payload, 0x00, 0x01, 0x00, 0x01)) + return true; + return false; + +} + +static inline bool match_upgrade_resp(uint32_t payload, uint32_t len) { + + if (len == 1 && MATCH(payload, 0x0a, 0x00, 0x00, 0x00)) + return true; + return false; + +} + +static inline bool match_netmfp(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* Limit to port 7150 for now -- this probably only corresponds to + * one MS service that uses this protocol, but I'm going to play + * it conservative. + */ + + if (data->server_port != 7150 && data->client_port != 7150) + return false; + + if (match_version_record(data->payload[0], data->payload_len[0])) { + if (match_upgrade_resp(data->payload[1], data->payload_len[1])) + return true; + } + + return false; +} + +static lpi_module_t lpi_netmfp = { + LPI_PROTO_NET_MFP, + LPI_CATEGORY_WEB, + "NET-MFP", + 199, + match_netmfp +}; + +void register_netmfp(LPIModuleMap *mod_map) { + register_protocol(&lpi_netmfp, mod_map); +} + diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index 2c89b52b..ae5a53bf 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -148,6 +148,7 @@ void register_ncsoft(LPIModuleMap *mod_map); void register_ndt_tput(LPIModuleMap *mod_map); void register_netbios(LPIModuleMap *mod_map); void register_netcat_cctv(LPIModuleMap *mod_map); +void register_netmfp(LPIModuleMap *mod_map); void register_nntp(LPIModuleMap *mod_map); void register_nntps(LPIModuleMap *mod_map); void register_norton_backup(LPIModuleMap *mod_map); diff --git a/lib/udp/lpi_zoom.cc b/lib/udp/lpi_zoom.cc index f4802339..b02cb32e 100644 --- a/lib/udp/lpi_zoom.cc +++ b/lib/udp/lpi_zoom.cc @@ -57,6 +57,8 @@ static inline bool match_zoom_02(uint32_t payload, uint32_t len) { static inline bool match_zoom_05(uint32_t payload) { if (MATCH(payload, 0x05, 0x10, 0x01, 0x00)) return true; + if (MATCH(payload, 0x05, 0x0f, 0x01, 0x00)) + return true; return false; } From 6471346d96e522f35787f102f497e81440ecb504 Mon Sep 17 00:00:00 2001 From: Jeroen Roovers Date: Mon, 12 Jun 2017 06:49:01 +0200 Subject: [PATCH 53/68] You allocate 750 bytes and then write 1000 bytes Obviously this should be optimised. :) --- tools/protoident/lpi_protoident.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/protoident/lpi_protoident.cc b/tools/protoident/lpi_protoident.cc index 35b47112..7af4f112 100644 --- a/tools/protoident/lpi_protoident.cc +++ b/tools/protoident/lpi_protoident.cc @@ -214,7 +214,7 @@ char *display_ident(Flow *f, IdentFlow *ident, struct globalopts *opts) { dump_payload(ident->lpi, 0, pload_out, 500); dump_payload(ident->lpi, 1, pload_in, 500); str = (char *)malloc(750); - snprintf(str, 1000, "%s %s %s %u %u %u %.3f %.3f %" PRIu64 " %" PRIu64 " %s %s\n", + snprintf(str, 750, "%s %s %s %u %u %u %.3f %.3f %" PRIu64 " %" PRIu64 " %s %s\n", proto->name, s_ip, c_ip, f->id.get_server_port(), f->id.get_client_port(), f->id.get_protocol(), ident->start_ts, From e36284a61c977d6a9bd9d24f457e81e43de145b9 Mon Sep 17 00:00:00 2001 From: Jeroen Roovers Date: Mon, 12 Jun 2017 06:51:57 +0200 Subject: [PATCH 54/68] You allocate 750 bytes and then write 1000 bytes .. --- tools/find_unknown/lpi_find_unknown.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/find_unknown/lpi_find_unknown.cc b/tools/find_unknown/lpi_find_unknown.cc index 27193142..37efd8f4 100644 --- a/tools/find_unknown/lpi_find_unknown.cc +++ b/tools/find_unknown/lpi_find_unknown.cc @@ -219,7 +219,7 @@ char *display_ident(Flow *f, IdentFlow *ident, struct globalopts *opts) { dump_payload(ident->lpi, 0, pload_out, 500); dump_payload(ident->lpi, 1, pload_in, 500); str = (char *)malloc(750); - snprintf(str, 1000, "%s %s %u %u %u %.3f %.3f %" PRIu64 " %" PRIu64 " %s %s\n", + snprintf(str, 750, "%s %s %u %u %u %.3f %.3f %" PRIu64 " %" PRIu64 " %s %s\n", s_ip, c_ip, f->id.get_server_port(), f->id.get_client_port(), f->id.get_protocol(), ident->start_ts, From e1768ead626703f88a135e0c2af10aa9f79b6fc2 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Tue, 13 Jun 2017 16:58:59 +1200 Subject: [PATCH 55/68] Added one new protocol: Speedin VPN Also tweaked a few other protocol rules: * Ramsey Dash * Xunlei * OpenVPN * PPLive --- lib/libprotoident.h | 1 + lib/proto_manager.cc | 1 + lib/tcp/Makefile.am | 1 + lib/tcp/lpi_speedin.cc | 83 +++++++++++++++++++++++++++++++++++++++ lib/tcp/tcp_protocols.h | 1 + lib/udp/lpi_openvpn.cc | 31 +++++++++++++++ lib/udp/lpi_pplive.cc | 32 +++++++-------- lib/udp/lpi_ramseydash.cc | 2 +- lib/udp/lpi_xunlei.cc | 19 ++++++++- 9 files changed, 153 insertions(+), 18 deletions(-) create mode 100644 lib/tcp/lpi_speedin.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 76796d61..64b02176 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -345,6 +345,7 @@ typedef enum { LPI_PROTO_TANKIX, LPI_PROTO_IPSHARKK, LPI_PROTO_NET_MFP, + LPI_PROTO_SPEEDIN, /* UDP Protocols */ LPI_PROTO_UDP, diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index 25d54f02..38ae1e46 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -236,6 +236,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_socks4(mod_map); register_socks5(mod_map); register_spdy(mod_map); + register_speedin(mod_map); register_speedtest(mod_map); register_spotify(mod_map); register_ssh(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index 018b2870..b9d5b5ce 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -169,6 +169,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_socks4.cc \ lpi_socks5.cc \ lpi_spdy.cc \ + lpi_speedin.cc \ lpi_speedtest.cc \ lpi_spotify.cc \ lpi_ssh.cc \ diff --git a/lib/tcp/lpi_speedin.cc b/lib/tcp/lpi_speedin.cc new file mode 100644 index 00000000..b9ce9f17 --- /dev/null +++ b/lib/tcp/lpi_speedin.cc @@ -0,0 +1,83 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* Speedin a.k.a. InVPN -- VPN for accessing Chinese content from outside + * of China. + */ +static inline bool match_speedin_3byte(uint32_t payload, uint32_t len) { + + if (len == 3 && MATCH(payload, 0x00, 0x00, 0x00, 0x00)) + return true; + return false; +} + +static inline bool match_speedin_other(uint32_t payload, uint32_t len) { + if (len <= 90 || len >= 115) + return false; + + if (MATCH(payload, 0x23, 0x00, ANY, ANY)) + return true; + if (MATCH(payload, 0x03, 0x00, ANY, ANY)) + return true; + return false; +} + +static inline bool match_speedin(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (data->server_port != 12000 && data->client_port != 12000) + return false; + + if (match_speedin_3byte(data->payload[0], data->payload_len[0])) { + if (match_speedin_other(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_speedin_3byte(data->payload[1], data->payload_len[1])) { + if (match_speedin_other(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_speedin = { + LPI_PROTO_SPEEDIN, + LPI_CATEGORY_TUNNELLING, + "Speedin", + 22, + match_speedin +}; + +void register_speedin(LPIModuleMap *mod_map) { + register_protocol(&lpi_speedin, mod_map); +} + diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index ae5a53bf..6aa75524 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -197,6 +197,7 @@ void register_smtps(LPIModuleMap *mod_map); void register_socks4(LPIModuleMap *mod_map); void register_socks5(LPIModuleMap *mod_map); void register_spdy(LPIModuleMap *mod_map); +void register_speedin(LPIModuleMap *mod_map); void register_speedtest(LPIModuleMap *mod_map); void register_spotify(LPIModuleMap *mod_map); void register_ssh(LPIModuleMap *mod_map); diff --git a/lib/udp/lpi_openvpn.cc b/lib/udp/lpi_openvpn.cc index dcbaf77f..9ff68271 100644 --- a/lib/udp/lpi_openvpn.cc +++ b/lib/udp/lpi_openvpn.cc @@ -72,6 +72,24 @@ static inline bool match_tunnelbear_38(uint32_t payload, uint32_t len) { } +static inline bool match_wscribe_40(uint32_t payload, uint32_t len) { + if (!MATCH(payload, 0x40, ANY, ANY, ANY)) + return false; + if (len != 98) + return false; + return true; + +} + +static inline bool match_wscribe_38(uint32_t payload, uint32_t len) { + if (!MATCH(payload, 0x38, ANY, ANY, ANY)) + return false; + if (len != 86) + return false; + return true; + +} + static inline bool match_openvpn_udp(lpi_data_t *data, lpi_module_t *mod UNUSED) { /* The payload matching alone isn't very strong, so I'm going to @@ -98,6 +116,19 @@ static inline bool match_openvpn_udp(lpi_data_t *data, lpi_module_t *mod UNUSED) } + /* Similar for Windscribe */ + if (data->server_port == 443 || data->client_port == 443) { + if (match_wscribe_40(data->payload[0], data->payload_len[0])) { + if (match_wscribe_38(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_wscribe_40(data->payload[1], data->payload_len[1])) { + if (match_wscribe_38(data->payload[0], data->payload_len[0])) + return true; + } + } + return false; } diff --git a/lib/udp/lpi_pplive.cc b/lib/udp/lpi_pplive.cc index 42254279..aaabde98 100644 --- a/lib/udp/lpi_pplive.cc +++ b/lib/udp/lpi_pplive.cc @@ -30,19 +30,21 @@ #include "proto_manager.h" #include "proto_common.h" -static inline bool obs_pplive_req(uint32_t len) { +static inline bool obs_pplive_req(uint32_t payload, uint32_t len) { /* There's always a 94 byte packet involved */ + if (payload == 0) + return false; if (len == 94) return true; return false; } -static inline bool obs_pplive_resp(uint32_t len) { - if (len == 0) +static inline bool obs_pplive_resp(uint32_t len, bool knownport) { + if (len == 0 && knownport) return true; if (len == 94) return true; - if (len == 49) + if (len == 49 && knownport) return true; return false; } @@ -51,24 +53,22 @@ static inline bool obs_pplive_resp(uint32_t len) { static inline bool match_obscure_pplive(lpi_data_t *data) { /* This is pretty tough stuff to match - the 4 bytes of payload - * is random, but the packet sizes seem consistent. It also seems to - * only occur on certain ports. + * is random, but the packet sizes seem consistent. * * DPI tools suggest this traffic is pplive, so we'll go with that * in the absence of any other documentation :/ */ + bool knownport = false; - /* Port 5041 and 8303 are the ports that this traffic is - * typically seen on */ - if (data->server_port != 5041 && data->server_port != 8303 && - data->client_port != 5041 && data->client_port != 8303) - return false; + /* Restrict non-94 byte responses to port 5041 */ + if (data->server_port == 5041 || data->client_port == 5041) + knownport = true; - if (obs_pplive_req(data->payload_len[0]) && - obs_pplive_resp(data->payload_len[1])) + if (obs_pplive_req(data->payload[0], data->payload_len[0]) && + obs_pplive_resp(data->payload_len[1], knownport)) return true; - if (obs_pplive_req(data->payload_len[1]) && - obs_pplive_resp(data->payload_len[0])) + if (obs_pplive_req(data->payload[1], data->payload_len[1]) && + obs_pplive_resp(data->payload_len[0], knownport)) return true; return false; @@ -104,7 +104,7 @@ static lpi_module_t lpi_pplive = { LPI_PROTO_UDP_PPLIVE, LPI_CATEGORY_P2PTV, "PPLive", - 3, + 203, match_pplive }; diff --git a/lib/udp/lpi_ramseydash.cc b/lib/udp/lpi_ramseydash.cc index 3f5a3400..28797de3 100644 --- a/lib/udp/lpi_ramseydash.cc +++ b/lib/udp/lpi_ramseydash.cc @@ -42,7 +42,7 @@ static inline bool match_rdash_56da(uint32_t payload, uint32_t len) { static inline bool match_rdash_da57(uint32_t payload, uint32_t len) { - if (len >= 150 && len <= 200 && MATCH(payload, 0xda, 0x57, ANY, ANY)) + if (len >= 140 && len <= 230 && MATCH(payload, 0xda, 0x57, ANY, ANY)) return true; return false; diff --git a/lib/udp/lpi_xunlei.cc b/lib/udp/lpi_xunlei.cc index 78593c56..b40d0e63 100644 --- a/lib/udp/lpi_xunlei.cc +++ b/lib/udp/lpi_xunlei.cc @@ -92,6 +92,23 @@ static inline bool match_xunlei_udp(lpi_data_t *data, lpi_module_t *mod UNUSED) } + /* Traffic seen while operating the Thunder client, not sure on exact + * purpose but can lead to large flows. Rule is not very strong, since + * the payload seems random. + */ + if (data->server_port == 12345 || data->client_port == 12345) { + if (data->payload[0] != 0 && data->payload[1] != 0) { + if (data->payload_len[0] >= 39) { + if (data->payload_len[0] <= 43) { + if (data->payload_len[1] >= 39) { + if (data->payload_len[1] <= 43) + return true; + } + } + } + } + } + if (match_str_both(data, "\x32\x00\x00\x00", "\x32\x00\x00\x00")) return true; if (match_str_both(data, "\x36\x00\x00\x00", "\x36\x00\x00\x00")) @@ -157,7 +174,7 @@ static lpi_module_t lpi_xunlei_udp = { LPI_PROTO_UDP_XUNLEI, LPI_CATEGORY_P2P, "Xunlei_UDP", - 3, + 203, match_xunlei_udp }; From 69109480253e58cc2cc3a80efa54843b2d18275d Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Wed, 14 Jun 2017 17:25:42 +1200 Subject: [PATCH 56/68] Updated a couple of protocols: ARMA 3 and mDNS --- lib/udp/lpi_arma3_server.cc | 11 +++++++---- lib/udp/lpi_mdns.cc | 23 +++++++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/lib/udp/lpi_arma3_server.cc b/lib/udp/lpi_arma3_server.cc index ffae4d99..29c5ed09 100644 --- a/lib/udp/lpi_arma3_server.cc +++ b/lib/udp/lpi_arma3_server.cc @@ -37,14 +37,17 @@ static inline bool match_arma3_server_payload(uint32_t payload, uint32_t len) { if (len == 0) return true; - if (!MATCH(payload, ANY, ANY, 0xe2, 0x16)) - return false; - replen = (payload & 0xffff); if (replen != len) return false; - return true; + if (MATCH(payload, ANY, ANY, 0xe2, 0x16)) + return true; + if (MATCH(payload, ANY, ANY, 0x60, 0xcf)) + return true; + + + return false; } diff --git a/lib/udp/lpi_mdns.cc b/lib/udp/lpi_mdns.cc index 3c904e74..32445ce0 100644 --- a/lib/udp/lpi_mdns.cc +++ b/lib/udp/lpi_mdns.cc @@ -32,8 +32,31 @@ /* Multicast DNS */ +static inline bool match_unicast_mdns(lpi_data_t *data) { + if (data->server_port != 5353 && data->client_port != 5353) { + return false; + } + + /* Only time I've ever seen this is Russian hackers trying + * to get more info about the local network. */ + + if (MATCH(data->payload[0], 0x00, 0x00, 0x84, 0x00)) { + if (MATCH(data->payload[1], 0x00, 0x00, 0x00, 0x00)) + return true; + } + + if (MATCH(data->payload[1], 0x00, 0x00, 0x84, 0x00)) { + if (MATCH(data->payload[0], 0x00, 0x00, 0x00, 0x00)) + return true; + } + return false; +} + static inline bool match_mdns(lpi_data_t *data, lpi_module_t *mod UNUSED) { + if (match_unicast_mdns(data)) + return true; + if (data->server_port != 5353) return false; if (data->client_port != 5353) From ca334a9fdc82297763a33f8af3d1051651b3666c Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Wed, 14 Jun 2017 17:26:09 +1200 Subject: [PATCH 57/68] Added another new protocol: Rising Storm 2 --- lib/libprotoident.h | 1 + lib/proto_manager.cc | 1 + lib/udp/Makefile.am | 1 + lib/udp/lpi_risingstorm.cc | 103 +++++++++++++++++++++++++++++++++++++ lib/udp/udp_protocols.h | 1 + 5 files changed, 107 insertions(+) create mode 100644 lib/udp/lpi_risingstorm.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 64b02176..050ccc1b 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -540,6 +540,7 @@ typedef enum { LPI_PROTO_UDP_GOTOMEETING, LPI_PROTO_UDP_CROSSOUT, LPI_PROTO_UDP_UMEYE, + LPI_PROTO_UDP_RISING_STORM, /* Patterns that we can match, but do not know the protocol */ LPI_PROTO_REJECTION, /* All responses are 0x02 */ diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index 38ae1e46..f1520bfa 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -445,6 +445,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_ramsey_dash(mod_map); register_rdp_udp(mod_map); register_real(mod_map); + register_risingstorm(mod_map); register_roblox(mod_map); register_robocraft(mod_map); register_rtcp(mod_map); diff --git a/lib/udp/Makefile.am b/lib/udp/Makefile.am index 54cefaf4..40668229 100644 --- a/lib/udp/Makefile.am +++ b/lib/udp/Makefile.am @@ -143,6 +143,7 @@ libprotoident_udp_la_SOURCES = \ lpi_ramseydash.cc \ lpi_rdp_udp.cc \ lpi_real.cc \ + lpi_risingstorm.cc \ lpi_roblox.cc \ lpi_robocraft.cc \ lpi_rtcp.cc \ diff --git a/lib/udp/lpi_risingstorm.cc b/lib/udp/lpi_risingstorm.cc new file mode 100644 index 00000000..89ab4ef0 --- /dev/null +++ b/lib/udp/lpi_risingstorm.cc @@ -0,0 +1,103 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* More specifically, this rule is based off Rising Storm 2 but it'll + * probably match other Tripwire games including the original. + * + * TBC against real game traffic, but that costs $$. + * All servers contacted were labelled as RS2 on various server tracking + * sites, so that's enough for me. + */ + +static inline bool match_rs_0080(uint32_t payload, uint32_t len) { + + if (MATCH(payload, 0x00, 0x80, 0x05, 0x20)) { + if (len == 10 || len == 17) + return true; + } + return false; + +} + +static inline bool match_rs_00c0(uint32_t payload, uint32_t len) { + + if (MATCH(payload, 0x00, 0xc0, ANY, 0x08)) { + if (len == 14) + return true; + } + return false; +} + +static inline bool match_rs_0108(uint32_t payload, uint32_t len) { + + if (MATCH(payload, 0x00, 0x00, 0x01, 0x08)) { + if (len == 25 || len == 12) + return true; + } + return false; +} + +static inline bool match_risingstorm(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_rs_0080(data->payload[0], data->payload_len[0])) { + if (match_rs_0108(data->payload[1], data->payload_len[1])) + return true; + + if (match_rs_00c0(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_rs_0080(data->payload[1], data->payload_len[1])) { + if (match_rs_0108(data->payload[0], data->payload_len[0])) + return true; + + if (match_rs_00c0(data->payload[0], data->payload_len[0])) + return true; + } + + + + return false; +} + +static lpi_module_t lpi_risingstorm = { + LPI_PROTO_UDP_RISING_STORM, + LPI_CATEGORY_GAMING, + "RisingStorm", + 12, + match_risingstorm +}; + +void register_risingstorm(LPIModuleMap *mod_map) { + register_protocol(&lpi_risingstorm, mod_map); +} + diff --git a/lib/udp/udp_protocols.h b/lib/udp/udp_protocols.h index 7464fa39..45042e90 100644 --- a/lib/udp/udp_protocols.h +++ b/lib/udp/udp_protocols.h @@ -172,6 +172,7 @@ void register_radius(LPIModuleMap *mod_map); void register_ramsey_dash(LPIModuleMap *mod_map); void register_rdp_udp(LPIModuleMap *mod_map); void register_real(LPIModuleMap *mod_map); +void register_risingstorm(LPIModuleMap *mod_map); void register_roblox(LPIModuleMap *mod_map); void register_robocraft(LPIModuleMap *mod_map); void register_rtcp(LPIModuleMap *mod_map); From d077e42f63d95df3ceb3728f4cb404a79ea182f5 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 16 Jun 2017 14:41:56 +1200 Subject: [PATCH 58/68] Added Crossfire protocols Game uses both TCP and UDP, with distinct protocols so 2 for the price of 1. --- lib/libprotoident.h | 2 + lib/proto_manager.cc | 4 +- lib/tcp/Makefile.am | 1 + lib/tcp/lpi_4d.cc | 22 +++++++++++ lib/tcp/lpi_crossfire_tcp.cc | 66 +++++++++++++++++++++++++++++++ lib/tcp/tcp_protocols.h | 1 + lib/udp/Makefile.am | 1 + lib/udp/lpi_crossfire.cc | 75 ++++++++++++++++++++++++++++++++++++ lib/udp/udp_protocols.h | 1 + 9 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 lib/tcp/lpi_crossfire_tcp.cc create mode 100644 lib/udp/lpi_crossfire.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 050ccc1b..97570e84 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -346,6 +346,7 @@ typedef enum { LPI_PROTO_IPSHARKK, LPI_PROTO_NET_MFP, LPI_PROTO_SPEEDIN, + LPI_PROTO_CROSSFIRE, /* UDP Protocols */ LPI_PROTO_UDP, @@ -541,6 +542,7 @@ typedef enum { LPI_PROTO_UDP_CROSSOUT, LPI_PROTO_UDP_UMEYE, LPI_PROTO_UDP_RISING_STORM, + LPI_PROTO_UDP_CROSSFIRE, /* Patterns that we can match, but do not know the protocol */ LPI_PROTO_REJECTION, /* All responses are 0x02 */ diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index f1520bfa..33d80727 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -93,6 +93,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_cod_waw(mod_map); register_conquer(mod_map); register_crashplan(mod_map); + register_crossfire_tcp(mod_map); register_cryptic(mod_map); register_cvs(mod_map); register_dell_backup(mod_map); @@ -147,7 +148,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_invalid_pop(mod_map); register_invalid_smtp(mod_map); register_ipop(mod_map); - register_ipsharkk(mod_map); + //register_ipsharkk(mod_map); register_irc(mod_map); register_java(mod_map); register_jedi(mod_map); @@ -330,6 +331,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_cirn(mod_map); register_cisco_ipsec(mod_map); register_cisco_sslvpn(mod_map); + register_crossfire(mod_map); register_crossout(mod_map); register_csgo(mod_map); register_db2(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index b9d5b5ce..3dd33158 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -26,6 +26,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_cod_waw.cc \ lpi_conquer.cc \ lpi_crashplan.cc \ + lpi_crossfire_tcp.cc \ lpi_cryptic.cc \ lpi_cvs.cc \ lpi_dell_backup.cc \ diff --git a/lib/tcp/lpi_4d.cc b/lib/tcp/lpi_4d.cc index 030a745e..078a9ba2 100644 --- a/lib/tcp/lpi_4d.cc +++ b/lib/tcp/lpi_4d.cc @@ -48,6 +48,18 @@ static inline bool match_4d_other(uint32_t payload, uint32_t len) { } +static inline bool match_4d_len72(uint32_t payload, uint32_t len) { + if (len == 72 && MATCH(payload, 0x00, 0x00, 0x00, 0x00)) + return true; + return false; +} + +static inline bool match_4d_len12(uint32_t payload, uint32_t len) { + if (len == 12 && MATCH(payload, 0x00, 0x00, 0x00, 0x00)) + return true; + return false; +} + static inline bool match_4d(lpi_data_t *data, lpi_module_t *mod UNUSED) { if (match_4d_5f(data->payload[0], data->payload_len[0])) { @@ -60,6 +72,16 @@ static inline bool match_4d(lpi_data_t *data, lpi_module_t *mod UNUSED) { return true; } + if (match_4d_len72(data->payload[0], data->payload_len[0])) { + if (match_4d_len12(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_4d_len72(data->payload[1], data->payload_len[1])) { + if (match_4d_len12(data->payload[0], data->payload_len[0])) + return true; + } + return false; } diff --git a/lib/tcp/lpi_crossfire_tcp.cc b/lib/tcp/lpi_crossfire_tcp.cc new file mode 100644 index 00000000..941c2b95 --- /dev/null +++ b/lib/tcp/lpi_crossfire_tcp.cc @@ -0,0 +1,66 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_cf_tcp(uint32_t payload, uint32_t len) { + + uint32_t hlen = (ntohl(payload) >> 8) & 0xffff; + + if (len < 1350 && hlen != len - 7) + return false; + if (MATCH(payload, 0xf1, ANY, ANY, 0x01)) + return true; + return false; +} + +static inline bool match_crossfire_tcp(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_cf_tcp(data->payload[0], data->payload_len[0])) { + if (match_cf_tcp(data->payload[1], data->payload_len[1])) { + return true; + } + } + + return false; +} + +static lpi_module_t lpi_crossfire_tcp = { + LPI_PROTO_CROSSFIRE, + LPI_CATEGORY_GAMING, + "CrossfireTCP", + 30, + match_crossfire_tcp +}; + +void register_crossfire_tcp(LPIModuleMap *mod_map) { + register_protocol(&lpi_crossfire_tcp, mod_map); +} + diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index 6aa75524..3db658bd 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -54,6 +54,7 @@ void register_clubbox(LPIModuleMap *mod_map); void register_cod_waw(LPIModuleMap *mod_map); void register_conquer(LPIModuleMap *mod_map); void register_crashplan(LPIModuleMap *mod_map); +void register_crossfire_tcp(LPIModuleMap *mod_map); void register_cryptic(LPIModuleMap *mod_map); void register_cvs(LPIModuleMap *mod_map); void register_dell_backup(LPIModuleMap *mod_map); diff --git a/lib/udp/Makefile.am b/lib/udp/Makefile.am index 40668229..70b2cdff 100644 --- a/lib/udp/Makefile.am +++ b/lib/udp/Makefile.am @@ -27,6 +27,7 @@ libprotoident_udp_la_SOURCES = \ lpi_cisco_ipsec.cc \ lpi_cisco_sslvpn.cc \ lpi_cod.cc \ + lpi_crossfire.cc \ lpi_crossout.cc \ lpi_csgo.cc \ lpi_db2.cc \ diff --git a/lib/udp/lpi_crossfire.cc b/lib/udp/lpi_crossfire.cc new file mode 100644 index 00000000..589a86fc --- /dev/null +++ b/lib/udp/lpi_crossfire.cc @@ -0,0 +1,75 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_cf_25(uint32_t payload, uint32_t len) { + + if (len == 25 && MATCH(payload, 0xc7, 0xd9, 0x19, 0x99)) + return true; + return false; + +} + +static inline bool match_cf_5(uint32_t payload, uint32_t len) { + + if (len == 5 && MATCH(payload, 0xc7, 0xd9, 0x19, 0x99)) + return true; + return false; + +} + +static inline bool match_crossfire(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_cf_25(data->payload[0], data->payload_len[0])) { + if (match_cf_5(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_cf_25(data->payload[1], data->payload_len[1])) { + if (match_cf_5(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_crossfire = { + LPI_PROTO_UDP_CROSSFIRE, + LPI_CATEGORY_GAMING, + "CrossfireUDP", + 10, + match_crossfire +}; + +void register_crossfire(LPIModuleMap *mod_map) { + register_protocol(&lpi_crossfire, mod_map); +} + diff --git a/lib/udp/udp_protocols.h b/lib/udp/udp_protocols.h index 45042e90..33b29a69 100644 --- a/lib/udp/udp_protocols.h +++ b/lib/udp/udp_protocols.h @@ -57,6 +57,7 @@ void register_chivalry(LPIModuleMap *mod_map); void register_cirn(LPIModuleMap *mod_map); void register_cisco_ipsec(LPIModuleMap *mod_map); void register_cisco_sslvpn(LPIModuleMap *mod_map); +void register_crossfire(LPIModuleMap *mod_map); void register_crossout(LPIModuleMap *mod_map); void register_csgo(LPIModuleMap *mod_map); void register_db2(LPIModuleMap *mod_map); From 763eecd59e3a1a12af4fe5cf2033c135210e518a Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 16 Jun 2017 15:23:47 +1200 Subject: [PATCH 59/68] Fix byte ordering issue with Crossfire TCP Also added a missing pattern - 0x00 is also allowed for the fourth byte. --- lib/tcp/lpi_crossfire_tcp.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/tcp/lpi_crossfire_tcp.cc b/lib/tcp/lpi_crossfire_tcp.cc index 941c2b95..8bed3fd0 100644 --- a/lib/tcp/lpi_crossfire_tcp.cc +++ b/lib/tcp/lpi_crossfire_tcp.cc @@ -32,12 +32,14 @@ static inline bool match_cf_tcp(uint32_t payload, uint32_t len) { - uint32_t hlen = (ntohl(payload) >> 8) & 0xffff; + uint32_t hlen = bswap_le_to_host32(payload & 0xffff00) >> 8; if (len < 1350 && hlen != len - 7) return false; if (MATCH(payload, 0xf1, ANY, ANY, 0x01)) return true; + if (MATCH(payload, 0xf1, ANY, ANY, 0x00)) + return true; return false; } From 6f95c363e83dcc2c6d1b77933672a58f79329836 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Mon, 19 Jun 2017 15:31:21 +1200 Subject: [PATCH 60/68] One new protocol: Dash Three updated protocols: Forticlient SSL VPN, Speedtest and Xunlei Re-enabled IPSharkk after erroneously disabling it in a previous commit. --- lib/libprotoident.h | 1 + lib/proto_manager.cc | 3 +- lib/tcp/Makefile.am | 1 + lib/tcp/lpi_dash.cc | 74 +++++++++++++++++++++++++++++++ lib/tcp/lpi_speedtest.cc | 21 +++++++++ lib/tcp/tcp_protocols.h | 1 + lib/udp/lpi_forticlient_sslvpn.cc | 2 +- lib/udp/lpi_xunlei.cc | 2 + 8 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 lib/tcp/lpi_dash.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 97570e84..badee425 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -347,6 +347,7 @@ typedef enum { LPI_PROTO_NET_MFP, LPI_PROTO_SPEEDIN, LPI_PROTO_CROSSFIRE, + LPI_PROTO_DASH, /* UDP Protocols */ LPI_PROTO_UDP, diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index 33d80727..a978c478 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -96,6 +96,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_crossfire_tcp(mod_map); register_cryptic(mod_map); register_cvs(mod_map); + register_dash(mod_map); register_dell_backup(mod_map); register_destiny(mod_map); register_diablo3(mod_map); @@ -148,7 +149,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_invalid_pop(mod_map); register_invalid_smtp(mod_map); register_ipop(mod_map); - //register_ipsharkk(mod_map); + register_ipsharkk(mod_map); register_irc(mod_map); register_java(mod_map); register_jedi(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index 3dd33158..1811f0b1 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -29,6 +29,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_crossfire_tcp.cc \ lpi_cryptic.cc \ lpi_cvs.cc \ + lpi_dash.cc \ lpi_dell_backup.cc \ lpi_destiny.cc \ lpi_diablo3.cc \ diff --git a/lib/tcp/lpi_dash.cc b/lib/tcp/lpi_dash.cc new file mode 100644 index 00000000..4355b072 --- /dev/null +++ b/lib/tcp/lpi_dash.cc @@ -0,0 +1,74 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_dash_125(uint32_t payload, uint32_t len) { + + if (len == 125 && MATCH(payload, 0xbf, 0x0c, 0x6b, 0xbd)) + return true; + return false; +} + +static inline bool match_dash_130(uint32_t payload, uint32_t len) { + + if (len == 130 && MATCH(payload, 0xbf, 0x0c, 0x6b, 0xbd)) + return true; + return false; +} + +static inline bool match_dash(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* Port 9999 */ + if (match_dash_125(data->payload[0], data->payload_len[0])) { + if (match_dash_130(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_dash_125(data->payload[1], data->payload_len[1])) { + if (match_dash_130(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_dash = { + LPI_PROTO_DASH, + LPI_CATEGORY_ECOMMERCE, + "Dash", + 5, + match_dash +}; + +void register_dash(LPIModuleMap *mod_map) { + register_protocol(&lpi_dash, mod_map); +} + diff --git a/lib/tcp/lpi_speedtest.cc b/lib/tcp/lpi_speedtest.cc index 6b96f4ce..ad9d561b 100644 --- a/lib/tcp/lpi_speedtest.cc +++ b/lib/tcp/lpi_speedtest.cc @@ -39,6 +39,14 @@ static inline bool match_speedtest_hi(uint32_t payload, uint32_t len) { return true; } +static inline bool match_speedtest_bighi(uint32_t payload, uint32_t len) { + + if (len == 40 && MATCH(payload, 'H', 'I', 0x20, '8')) + return true; + return false; + +} + static inline bool match_speedtest_hello(uint32_t payload, uint32_t len) { if (len == 0) @@ -55,12 +63,25 @@ static inline bool match_speedtest(lpi_data_t *data, lpi_module_t *mod UNUSED) { return true; } } + if (match_speedtest_hi(data->payload[1], data->payload_len[1])) { if (match_speedtest_hello(data->payload[0], data->payload_len[0])) { return true; } } + if (match_speedtest_bighi(data->payload[0], data->payload_len[0])) { + if (match_speedtest_hello(data->payload[1], data->payload_len[1])) { + return true; + } + } + + if (match_speedtest_bighi(data->payload[1], data->payload_len[1])) { + if (match_speedtest_hello(data->payload[0], data->payload_len[0])) { + return true; + } + } + return false; } diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index 3db658bd..526bc5f0 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -57,6 +57,7 @@ void register_crashplan(LPIModuleMap *mod_map); void register_crossfire_tcp(LPIModuleMap *mod_map); void register_cryptic(LPIModuleMap *mod_map); void register_cvs(LPIModuleMap *mod_map); +void register_dash(LPIModuleMap *mod_map); void register_dell_backup(LPIModuleMap *mod_map); void register_destiny(LPIModuleMap *mod_map); void register_diablo3(LPIModuleMap *mod_map); diff --git a/lib/udp/lpi_forticlient_sslvpn.cc b/lib/udp/lpi_forticlient_sslvpn.cc index e72c8dd1..5f052892 100644 --- a/lib/udp/lpi_forticlient_sslvpn.cc +++ b/lib/udp/lpi_forticlient_sslvpn.cc @@ -31,7 +31,7 @@ #include "proto_common.h" static inline bool match_forti_vpn_48(uint32_t payload, uint32_t len) { - if (len != 48) + if (len != 48 && len != 192) return false; if (MATCHSTR(payload, "\x16\xfe\xff\x00")) return true; diff --git a/lib/udp/lpi_xunlei.cc b/lib/udp/lpi_xunlei.cc index b40d0e63..960edad9 100644 --- a/lib/udp/lpi_xunlei.cc +++ b/lib/udp/lpi_xunlei.cc @@ -67,6 +67,8 @@ static inline bool match_shuijing_3b(uint32_t payload, uint32_t len) { return true; if (len == 31 && MATCH(payload, 0x3b, 0x00, 0x00, 0x00)) return true; + if (len == 29 && MATCH(payload, 0x3b, 0x00, 0x00, 0x00)) + return true; return false; } From e8ed4580a0ec8269b778d293b77b810f862cfc6c Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Wed, 21 Jun 2017 11:27:40 +1200 Subject: [PATCH 61/68] Add file signature for CoralReef traces --- lib/proto_common.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/proto_common.cc b/lib/proto_common.cc index 27e8638c..c041a659 100644 --- a/lib/proto_common.cc +++ b/lib/proto_common.cc @@ -317,6 +317,10 @@ bool match_file_header(uint32_t payload) { if (MATCH(payload, 'V', 'B', 'S', 'P')) return true; + /* Old coralreef trace files! */ + if (MATCHSTR(payload, "\xff\xff\x44\x00")) + return true; + /* I'm pretty sure the following are files of some type or another. * They crop up pretty often in our test data sets, so I'm going to * put them in here. From 998ae58cefbe5240e86acd8a8ee6d734b90e72d3 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Wed, 21 Jun 2017 11:27:56 +1200 Subject: [PATCH 62/68] Expand allowable packet sizes for Dogecoin --- lib/tcp/lpi_dogecoin.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/tcp/lpi_dogecoin.cc b/lib/tcp/lpi_dogecoin.cc index 84188870..d069002a 100644 --- a/lib/tcp/lpi_dogecoin.cc +++ b/lib/tcp/lpi_dogecoin.cc @@ -32,8 +32,10 @@ static inline bool match_dc_magic(uint32_t payload, uint32_t len) { - if (len == 129 && MATCH(payload, 0xc0, 0xc0, 0xc0, 0xc0)) - return true; + if (MATCH(payload, 0xc0, 0xc0, 0xc0, 0xc0)) { + if (len >= 125 && len <= 129) + return true; + } return false; } From 760c7ff92adeb7e3868f1047732e8b6b13fc2fe0 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Fri, 23 Jun 2017 17:20:22 +1200 Subject: [PATCH 63/68] Two new protocols: Meraki Cloud and AirMedia Also updated: Bitcoin Maple Story Minecraft DTLS --- configure.ac | 2 +- lib/Makefile.am | 2 +- lib/libprotoident.h | 2 + lib/proto_manager.cc | 2 + lib/tcp/Makefile.am | 1 + lib/tcp/lpi_airmedia.cc | 76 +++++++++++++++++++++++++++++++++ lib/tcp/lpi_bitcoin.cc | 2 +- lib/tcp/lpi_maplestory_china.cc | 2 + lib/tcp/lpi_minecraft.cc | 10 ++++- lib/tcp/tcp_protocols.h | 1 + lib/udp/Makefile.am | 1 + lib/udp/lpi_dtls.cc | 13 ++++++ lib/udp/lpi_merakicloud.cc | 61 ++++++++++++++++++++++++++ lib/udp/udp_protocols.h | 1 + 14 files changed, 172 insertions(+), 4 deletions(-) create mode 100644 lib/tcp/lpi_airmedia.cc create mode 100644 lib/udp/lpi_merakicloud.cc diff --git a/configure.ac b/configure.ac index afbdcbf2..5c70925c 100644 --- a/configure.ac +++ b/configure.ac @@ -1,4 +1,4 @@ -AC_INIT(libprotoident, 2.0.10, contact@wand.net.nz) +AC_INIT(libprotoident, 2.0.11, contact@wand.net.nz) AM_INIT_AUTOMAKE AC_CONFIG_SRCDIR(lib/libprotoident.cc) diff --git a/lib/Makefile.am b/lib/Makefile.am index 8f4ead54..f2da02e9 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -10,5 +10,5 @@ libprotoident_la_SOURCES=libprotoident.h libprotoident.cc \ INCLUDES=@ADD_INCLS@ libprotoident_la_LIBADD = @ADD_LIBS@ tcp/libprotoident_tcp.la \ udp/libprotoident_udp.la -libprotoident_la_LDFLAGS = @ADD_LDFLAGS@ -version-info 2:10:0 +libprotoident_la_LDFLAGS = @ADD_LDFLAGS@ -version-info 2:11:0 libprotoident_la_CPPFLAGS = -Werror diff --git a/lib/libprotoident.h b/lib/libprotoident.h index badee425..4b9f74f5 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -348,6 +348,7 @@ typedef enum { LPI_PROTO_SPEEDIN, LPI_PROTO_CROSSFIRE, LPI_PROTO_DASH, + LPI_PROTO_AIRMEDIA, /* UDP Protocols */ LPI_PROTO_UDP, @@ -544,6 +545,7 @@ typedef enum { LPI_PROTO_UDP_UMEYE, LPI_PROTO_UDP_RISING_STORM, LPI_PROTO_UDP_CROSSFIRE, + LPI_PROTO_UDP_MERAKICLOUD, /* Patterns that we can match, but do not know the protocol */ LPI_PROTO_REJECTION, /* All responses are 0x02 */ diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index a978c478..3c62d508 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -72,6 +72,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_acestream(mod_map); register_afp(mod_map); register_airdroid(mod_map); + register_airmedia(mod_map); register_akamai_tcp(mod_map); register_amp(mod_map); register_appearin(mod_map); @@ -396,6 +397,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_linkproof(mod_map); register_lol(mod_map); register_mdns(mod_map); + register_merakicloud(mod_map); register_moh(mod_map); register_mp2p_udp(mod_map); register_msn_cache(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index 1811f0b1..186b6a39 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -5,6 +5,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_acestream.cc \ lpi_afp.cc \ lpi_airdroid.cc \ + lpi_airmedia.cc \ lpi_akamai.cc \ lpi_amp.cc \ lpi_appearin.cc \ diff --git a/lib/tcp/lpi_airmedia.cc b/lib/tcp/lpi_airmedia.cc new file mode 100644 index 00000000..752b8fe5 --- /dev/null +++ b/lib/tcp/lpi_airmedia.cc @@ -0,0 +1,76 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* Crestron Airmedia -- more details at: + * http://www.boredhackerblog.info/2016/02/extracting-images-from-crestron.html + */ + +static inline bool match_cam_wppi(uint32_t payload, uint32_t len) { + if (len == 12 && MATCHSTR(payload, "wppi")) + return true; + return false; +} + +static inline bool match_cam_sender(uint32_t payload, uint32_t len) { + if (len == 32 && MATCHSTR(payload, "Send")) + return true; + return false; +} + +static inline bool match_airmedia(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* Port 515 */ + if (match_cam_wppi(data->payload[0], data->payload_len[0])) { + if (match_cam_sender(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_cam_wppi(data->payload[1], data->payload_len[1])) { + if (match_cam_sender(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_airmedia = { + LPI_PROTO_AIRMEDIA, + LPI_CATEGORY_REMOTE, + "Airmedia", + 5, + match_airmedia +}; + +void register_airmedia(LPIModuleMap *mod_map) { + register_protocol(&lpi_airmedia, mod_map); +} + diff --git a/lib/tcp/lpi_bitcoin.cc b/lib/tcp/lpi_bitcoin.cc index c93f5b84..137e78cc 100644 --- a/lib/tcp/lpi_bitcoin.cc +++ b/lib/tcp/lpi_bitcoin.cc @@ -33,7 +33,7 @@ static inline bool match_bc_version(uint32_t payload, uint32_t len) { - if (len == 24 && MATCH(payload, 0xf9, 0xbe, 0xb4, 0xd9)) + if (len >= 24 && MATCH(payload, 0xf9, 0xbe, 0xb4, 0xd9)) return true; return false; } diff --git a/lib/tcp/lpi_maplestory_china.cc b/lib/tcp/lpi_maplestory_china.cc index 8bc7dfe0..a0f97596 100644 --- a/lib/tcp/lpi_maplestory_china.cc +++ b/lib/tcp/lpi_maplestory_china.cc @@ -39,6 +39,8 @@ static inline bool match_cms_hello(uint32_t payload, uint32_t len) { return true; if (MATCH(payload, 0x0e, 0x00, 0x8f, 0x00)) return true; + if (MATCH(payload, 0x0e, 0x00, 0x90, 0x00)) + return true; } return false; diff --git a/lib/tcp/lpi_minecraft.cc b/lib/tcp/lpi_minecraft.cc index d84deeb6..0dab2c3d 100644 --- a/lib/tcp/lpi_minecraft.cc +++ b/lib/tcp/lpi_minecraft.cc @@ -78,10 +78,18 @@ static inline bool match_mc_handshake(uint32_t payload, uint32_t len) { replen = ntohl(payload) >> 24; if (replen == len - 1) { - if (MATCH(payload, ANY, 0x00, ANY, ANY)) + if (MATCH(payload, ANY, 0x00, ANY, ANY) && len - 1 <= 255) + return true; + if (MATCH(payload, ANY, 0x01, ANY, ANY) && len - 1 >= 256) return true; } + /* Some handshakes seem to be undersized? */ + if (len == 189 && MATCH(payload, 0xbb, 0x01, 0x01, 0x10)) + return true; + if (len == 190 && MATCH(payload, 0xbc, 0x01, 0x01, 0x11)) + return true; + return false; } diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index 526bc5f0..49da3e5c 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -33,6 +33,7 @@ void register_4d(LPIModuleMap *mod_map); void register_acestream(LPIModuleMap *mod_map); void register_afp(LPIModuleMap *mod_map); void register_airdroid(LPIModuleMap *mod_map); +void register_airmedia(LPIModuleMap *mod_map); void register_akamai_tcp(LPIModuleMap *mod_map); void register_amp(LPIModuleMap *mod_map); void register_appearin(LPIModuleMap *mod_map); diff --git a/lib/udp/Makefile.am b/lib/udp/Makefile.am index 70b2cdff..11284e38 100644 --- a/lib/udp/Makefile.am +++ b/lib/udp/Makefile.am @@ -91,6 +91,7 @@ libprotoident_udp_la_SOURCES = \ lpi_linkproof.cc \ lpi_lol.cc \ lpi_mdns.cc \ + lpi_merakicloud.cc \ lpi_moh.cc \ lpi_mp2p.cc \ lpi_msn_cache.cc \ diff --git a/lib/udp/lpi_dtls.cc b/lib/udp/lpi_dtls.cc index ed72296b..a8466c72 100644 --- a/lib/udp/lpi_dtls.cc +++ b/lib/udp/lpi_dtls.cc @@ -52,6 +52,19 @@ static inline bool match_dtls(lpi_data_t *data, lpi_module_t *mod UNUSED) { if (data->payload_len[0] == 0) return true; } + + if (MATCHSTR(data->payload[0], "\x16\xfe\xff\x00")) { + if (data->payload_len[1] == 0) + return true; + if (MATCHSTR(data->payload[1], "\x16\xfe\xff\x00")) + return true; + } + + if (MATCHSTR(data->payload[1], "\x16\xfe\xff\x00")) { + if (data->payload_len[0] == 0) + return true; + } + return false; } diff --git a/lib/udp/lpi_merakicloud.cc b/lib/udp/lpi_merakicloud.cc new file mode 100644 index 00000000..9e2045b7 --- /dev/null +++ b/lib/udp/lpi_merakicloud.cc @@ -0,0 +1,61 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + + +static inline bool match_merakicloud(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* Port 7351 */ + + /* This may just be a user id of some sort -- need to see multiple + * users to confirm this is a fixed pattern. + */ + if (MATCH(data->payload[0], 0xfe, 0xf7, 0x28, 0x91)) { + if (MATCH(data->payload[1], 0xfe, 0xf7, 0x28, 0x91)) { + return true; + } + } + + return false; +} + +static lpi_module_t lpi_merakicloud = { + LPI_PROTO_UDP_MERAKICLOUD, + LPI_CATEGORY_CLOUD, + "MerakiCloud", + 34, + match_merakicloud +}; + +void register_merakicloud(LPIModuleMap *mod_map) { + register_protocol(&lpi_merakicloud, mod_map); +} + diff --git a/lib/udp/udp_protocols.h b/lib/udp/udp_protocols.h index 33b29a69..6189d0e5 100644 --- a/lib/udp/udp_protocols.h +++ b/lib/udp/udp_protocols.h @@ -121,6 +121,7 @@ void register_line_udp(LPIModuleMap *mod_map); void register_linkproof(LPIModuleMap *mod_map); void register_lol(LPIModuleMap *mod_map); void register_mdns(LPIModuleMap *mod_map); +void register_merakicloud(LPIModuleMap *mod_map); void register_moh(LPIModuleMap *mod_map); void register_mp2p_udp(LPIModuleMap *mod_map); void register_msn_cache(LPIModuleMap *mod_map); From c6d8d6e6af5cdc5df06d25a0e0299fc118f88b7d Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Mon, 26 Jun 2017 11:57:32 +1200 Subject: [PATCH 64/68] Add support for large abridged Telegram queries --- lib/tcp/lpi_telegram.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/tcp/lpi_telegram.cc b/lib/tcp/lpi_telegram.cc index c4c897ae..704876fc 100644 --- a/lib/tcp/lpi_telegram.cc +++ b/lib/tcp/lpi_telegram.cc @@ -54,6 +54,15 @@ static inline bool match_abridged_telegram_query(uint32_t payload, uint32_t len) } + /* All 1s in the length field means the next three bytes are a + * length field. In this case, the first packet will almost + * certainly be MSS sized. + */ + if (MATCH(payload, 0xef, 0x7f, ANY, ANY)) { + if (len >= 1300) + return true; + } + return false; } From 32789ee8fcb430cd9de5cd6927f854d33f4aab41 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Wed, 28 Jun 2017 11:23:29 +1200 Subject: [PATCH 65/68] Three new protocols * Dahua * Snap VPN * GIOP Also tweaked Speedtest and Portmap RPC to match more traffic. --- lib/libprotoident.h | 3 ++ lib/proto_manager.cc | 3 ++ lib/tcp/Makefile.am | 1 + lib/tcp/lpi_giop.cc | 62 ++++++++++++++++++++++++++++++ lib/tcp/lpi_speedtest.cc | 2 +- lib/tcp/tcp_protocols.h | 1 + lib/udp/Makefile.am | 2 + lib/udp/lpi_dahua.cc | 63 +++++++++++++++++++++++++++++++ lib/udp/lpi_portmap_rpc.cc | 6 ++- lib/udp/lpi_snapvpn.cc | 77 ++++++++++++++++++++++++++++++++++++++ lib/udp/udp_protocols.h | 2 + 11 files changed, 219 insertions(+), 3 deletions(-) create mode 100644 lib/tcp/lpi_giop.cc create mode 100644 lib/udp/lpi_dahua.cc create mode 100644 lib/udp/lpi_snapvpn.cc diff --git a/lib/libprotoident.h b/lib/libprotoident.h index 4b9f74f5..956e0265 100644 --- a/lib/libprotoident.h +++ b/lib/libprotoident.h @@ -349,6 +349,7 @@ typedef enum { LPI_PROTO_CROSSFIRE, LPI_PROTO_DASH, LPI_PROTO_AIRMEDIA, + LPI_PROTO_GIOP, /* UDP Protocols */ LPI_PROTO_UDP, @@ -546,6 +547,8 @@ typedef enum { LPI_PROTO_UDP_RISING_STORM, LPI_PROTO_UDP_CROSSFIRE, LPI_PROTO_UDP_MERAKICLOUD, + LPI_PROTO_UDP_SNAPVPN, + LPI_PROTO_UDP_DAHUA, /* Patterns that we can match, but do not know the protocol */ LPI_PROTO_REJECTION, /* All responses are 0x02 */ diff --git a/lib/proto_manager.cc b/lib/proto_manager.cc index 3c62d508..e92dabb7 100644 --- a/lib/proto_manager.cc +++ b/lib/proto_manager.cc @@ -122,6 +122,7 @@ int register_tcp_protocols(LPIModuleMap *mod_map) { register_fuckcoin(mod_map); register_funshion_tcp(mod_map); register_gamespy_tcp(mod_map); + register_giop(mod_map); register_git(mod_map); register_glupteba(mod_map); register_gnutella(mod_map); @@ -336,6 +337,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_crossfire(mod_map); register_crossout(mod_map); register_csgo(mod_map); + register_dahua(mod_map); register_db2(mod_map); register_dcc_udp(mod_map); register_demonware(mod_map); @@ -462,6 +464,7 @@ int register_udp_protocols(LPIModuleMap *mod_map) { register_sip_udp(mod_map); register_skype(mod_map); register_slp(mod_map); + register_snapvpn(mod_map); register_snmp(mod_map); register_sopcast(mod_map); register_spamfighter(mod_map); diff --git a/lib/tcp/Makefile.am b/lib/tcp/Makefile.am index 186b6a39..7662de9e 100644 --- a/lib/tcp/Makefile.am +++ b/lib/tcp/Makefile.am @@ -55,6 +55,7 @@ libprotoident_tcp_la_SOURCES = \ lpi_fuckcoin.cc \ lpi_funshion.cc \ lpi_gamespy.cc \ + lpi_giop.cc \ lpi_git.cc \ lpi_glupteba.cc \ lpi_gnutella.cc \ diff --git a/lib/tcp/lpi_giop.cc b/lib/tcp/lpi_giop.cc new file mode 100644 index 00000000..8dc926dd --- /dev/null +++ b/lib/tcp/lpi_giop.cc @@ -0,0 +1,62 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + + +static inline bool match_giop(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (MATCH(data->payload[0], 'G', 'I', 'O', 'P')) { + if (MATCH(data->payload[1], 'G', 'I', 'O', 'P')) + return true; + if (data->payload_len[1] == 0) + return true; + } + + if (MATCH(data->payload[1], 'G', 'I', 'O', 'P')) { + if (data->payload_len[0] == 0) + return true; + } + + return false; +} + +static lpi_module_t lpi_giop = { + LPI_PROTO_GIOP, + LPI_CATEGORY_REMOTE, + "GIOP", + 5, + match_giop +}; + +void register_giop(LPIModuleMap *mod_map) { + register_protocol(&lpi_giop, mod_map); +} + diff --git a/lib/tcp/lpi_speedtest.cc b/lib/tcp/lpi_speedtest.cc index ad9d561b..d8425ca7 100644 --- a/lib/tcp/lpi_speedtest.cc +++ b/lib/tcp/lpi_speedtest.cc @@ -41,7 +41,7 @@ static inline bool match_speedtest_hi(uint32_t payload, uint32_t len) { static inline bool match_speedtest_bighi(uint32_t payload, uint32_t len) { - if (len == 40 && MATCH(payload, 'H', 'I', 0x20, '8')) + if (len == 40 && MATCH(payload, 'H', 'I', 0x20, ANY)) return true; return false; diff --git a/lib/tcp/tcp_protocols.h b/lib/tcp/tcp_protocols.h index 49da3e5c..71beea59 100644 --- a/lib/tcp/tcp_protocols.h +++ b/lib/tcp/tcp_protocols.h @@ -83,6 +83,7 @@ void register_ftpdata(LPIModuleMap *mod_map); void register_fuckcoin(LPIModuleMap *mod_map); void register_funshion_tcp(LPIModuleMap *mod_map); void register_gamespy_tcp(LPIModuleMap *mod_map); +void register_giop(LPIModuleMap *mod_map); void register_git(LPIModuleMap *mod_map); void register_glupteba(LPIModuleMap *mod_map); void register_gnutella(LPIModuleMap *mod_map); diff --git a/lib/udp/Makefile.am b/lib/udp/Makefile.am index 11284e38..11a69267 100644 --- a/lib/udp/Makefile.am +++ b/lib/udp/Makefile.am @@ -30,6 +30,7 @@ libprotoident_udp_la_SOURCES = \ lpi_crossfire.cc \ lpi_crossout.cc \ lpi_csgo.cc \ + lpi_dahua.cc \ lpi_db2.cc \ lpi_dcc.cc \ lpi_demonware.cc \ @@ -157,6 +158,7 @@ libprotoident_udp_la_SOURCES = \ lpi_sip.cc \ lpi_skype.cc \ lpi_slp.cc \ + lpi_snapvpn.cc \ lpi_snmp.cc \ lpi_sopcast.cc \ lpi_spamfighter.cc \ diff --git a/lib/udp/lpi_dahua.cc b/lib/udp/lpi_dahua.cc new file mode 100644 index 00000000..5747a71d --- /dev/null +++ b/lib/udp/lpi_dahua.cc @@ -0,0 +1,63 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +/* Chinese IP surveillance Cameras */ + +static inline bool match_dahua_p2p(uint32_t payload, uint32_t len) { + if (len == 44 && MATCHSTR(payload, "\xff\xfe\xff\xe7")) + return true; + return false; + +} + +static inline bool match_dahua(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + if (match_dahua_p2p(data->payload[0], data->payload_len[0])) { + if (match_dahua_p2p(data->payload[1], data->payload_len[1])) + return true; + } + + return false; +} + +static lpi_module_t lpi_dahua = { + LPI_PROTO_UDP_DAHUA, + LPI_CATEGORY_STREAMING, + "Dahua", + 13, + match_dahua +}; + +void register_dahua(LPIModuleMap *mod_map) { + register_protocol(&lpi_dahua, mod_map); +} + diff --git a/lib/udp/lpi_portmap_rpc.cc b/lib/udp/lpi_portmap_rpc.cc index 563c315d..79dfdd53 100644 --- a/lib/udp/lpi_portmap_rpc.cc +++ b/lib/udp/lpi_portmap_rpc.cc @@ -36,8 +36,10 @@ static inline bool match_portmap_rpc(lpi_data_t *data, lpi_module_t *mod UNUSED) if (data->server_port != 111 && data->client_port != 111) return false; - if (data->payload[0] != data->payload[1]) - return false; + if (data->payload_len[0] > 0 && data->payload_len[1] > 0) { + if (data->payload[0] != data->payload[1]) + return false; + } if (data->payload_len[0] == 46 || data->payload_len[1] == 46) return true; diff --git a/lib/udp/lpi_snapvpn.cc b/lib/udp/lpi_snapvpn.cc new file mode 100644 index 00000000..e5ab71fe --- /dev/null +++ b/lib/udp/lpi_snapvpn.cc @@ -0,0 +1,77 @@ +/* + * + * Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. + * All rights reserved. + * + * This file is part of libprotoident. + * + * This code has been developed by the University of Waikato WAND + * research group. For further information please see http://www.wand.net.nz/ + * + * libprotoident is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * libprotoident is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + * + */ + +#include + +#include "libprotoident.h" +#include "proto_manager.h" +#include "proto_common.h" + +static inline bool match_snap_query(uint32_t payload, uint32_t len) { + + if (MATCH(payload, '2', '7', '3', 'P') && len < 50) + return true; + return false; +} + +static inline bool match_snap_fail(uint32_t payload, uint32_t len) { + /* Saw this a lot when failing to connect to the server */ + if (len == 14 && MATCH(payload, 0x0a, ANY, ANY, ANY)) + return true; + return false; +} + +static inline bool match_snapvpn(lpi_data_t *data, lpi_module_t *mod UNUSED) { + + /* Ports 21, 25, 109, 119 */ + + if (match_snap_query(data->payload[0], data->payload_len[0])) { + if (match_snap_query(data->payload[1], data->payload_len[1])) + return true; + if (match_snap_fail(data->payload[1], data->payload_len[1])) + return true; + } + + if (match_snap_query(data->payload[1], data->payload_len[1])) { + if (match_snap_fail(data->payload[0], data->payload_len[0])) + return true; + } + + return false; +} + +static lpi_module_t lpi_snapvpn = { + LPI_PROTO_UDP_SNAPVPN, + LPI_CATEGORY_TUNNELLING, + "SnapVPN", + 25, + match_snapvpn +}; + +void register_snapvpn(LPIModuleMap *mod_map) { + register_protocol(&lpi_snapvpn, mod_map); +} + diff --git a/lib/udp/udp_protocols.h b/lib/udp/udp_protocols.h index 6189d0e5..175e54e8 100644 --- a/lib/udp/udp_protocols.h +++ b/lib/udp/udp_protocols.h @@ -60,6 +60,7 @@ void register_cisco_sslvpn(LPIModuleMap *mod_map); void register_crossfire(LPIModuleMap *mod_map); void register_crossout(LPIModuleMap *mod_map); void register_csgo(LPIModuleMap *mod_map); +void register_dahua(LPIModuleMap *mod_map); void register_db2(LPIModuleMap *mod_map); void register_dcc_udp(LPIModuleMap *mod_map); void register_demonware(LPIModuleMap *mod_map); @@ -186,6 +187,7 @@ void register_serialnumberd(LPIModuleMap *mod_map); void register_sip_udp(LPIModuleMap *mod_map); void register_skype(LPIModuleMap *mod_map); void register_slp(LPIModuleMap *mod_map); +void register_snapvpn(LPIModuleMap *mod_map); void register_snmp(LPIModuleMap *mod_map); void register_sopcast(LPIModuleMap *mod_map); void register_spamfighter(LPIModuleMap *mod_map); From 91951797f6545b7a27ae661ed8167949e73b92fb Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Wed, 28 Jun 2017 13:01:39 +1200 Subject: [PATCH 66/68] Also fix lengths in calls to dump_payload() --- tools/find_unknown/lpi_find_unknown.cc | 4 ++-- tools/protoident/lpi_protoident.cc | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/find_unknown/lpi_find_unknown.cc b/tools/find_unknown/lpi_find_unknown.cc index 37efd8f4..99df465a 100644 --- a/tools/find_unknown/lpi_find_unknown.cc +++ b/tools/find_unknown/lpi_find_unknown.cc @@ -216,8 +216,8 @@ char *display_ident(Flow *f, IdentFlow *ident, struct globalopts *opts) { f->id.get_server_ip_str(s_ip); f->id.get_client_ip_str(c_ip); - dump_payload(ident->lpi, 0, pload_out, 500); - dump_payload(ident->lpi, 1, pload_in, 500); + dump_payload(ident->lpi, 0, pload_out, 100); + dump_payload(ident->lpi, 1, pload_in, 100); str = (char *)malloc(750); snprintf(str, 750, "%s %s %u %u %u %.3f %.3f %" PRIu64 " %" PRIu64 " %s %s\n", s_ip, c_ip, diff --git a/tools/protoident/lpi_protoident.cc b/tools/protoident/lpi_protoident.cc index 7af4f112..2bae06cd 100644 --- a/tools/protoident/lpi_protoident.cc +++ b/tools/protoident/lpi_protoident.cc @@ -211,8 +211,8 @@ char *display_ident(Flow *f, IdentFlow *ident, struct globalopts *opts) { f->id.get_server_ip_str(s_ip); f->id.get_client_ip_str(c_ip); - dump_payload(ident->lpi, 0, pload_out, 500); - dump_payload(ident->lpi, 1, pload_in, 500); + dump_payload(ident->lpi, 0, pload_out, 100); + dump_payload(ident->lpi, 1, pload_in, 100); str = (char *)malloc(750); snprintf(str, 750, "%s %s %s %u %u %u %.3f %.3f %" PRIu64 " %" PRIu64 " %s %s\n", proto->name, s_ip, c_ip, From ee41eacf7130cd5498e885f3a7b11e5a0db93970 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Wed, 28 Jun 2017 13:11:54 +1200 Subject: [PATCH 67/68] Updated README for 2.0.11 release Added Jeroen to list of contributors. Updated references to libtrace and libflowmanager to reflect new minimum requirements. --- README | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/README b/README index 306407c9..f248447c 100644 --- a/README +++ b/README @@ -1,7 +1,7 @@ -libprotoident 2.0.10 +libprotoident 2.0.11 --------------------------------------------------------------------------- -Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand. +Copyright (c) 2011-2017 The University of Waikato, Hamilton, New Zealand. All rights reserved. This code has been developed by the University of Waikato WAND @@ -24,6 +24,7 @@ With contributions from: Aaron Murrihy Paweł Foremski Fabian Weisshaar + Jeroen Roovers Introduction ============ @@ -36,10 +37,10 @@ some tools that can be used to perform simple analysis of traffic flows. Required Libraries ================== -libtrace +libtrace 4.0.1 or later * available from http://research.wand.net.nz/software/libtrace.php -libflowmanager 2.0.4 or later +libflowmanager 3.0.0 or later * optional, but required to build the tools * available from http://research.wand.net.nz/software/libflowmanager.php @@ -208,7 +209,7 @@ There are currently four tools included with libprotoident. * Flow start time (as a Unix timestamp) * lpi_live (DEPRECATED) - + Description: This tool reports byte and packet counts (both inbound and outbound) for each identified protocol in real-time. Identification of a flow @@ -261,7 +262,7 @@ There are currently four tools included with libprotoident. - out_curr_flows = outbound flows active at the period end * The application protocol being measured * The value for the measured statistic - + API === @@ -270,15 +271,15 @@ use the libprotoident API. The API is very simple and the best way to learn it is to examine how the existing tools work. The source for the tools is located in the tools/ directory. -The tools use libflowmanager to do the flow tracking, using functions beginning -with 'lfm_'. You will probably want to incorporate this into your own tool. -Usage of libprotoident itself is through functions beginning with 'lpi_'. +The tools use libflowmanager to do the flow tracking, using an instance of a +FlowManager class. You will probably want to incorporate this into your own +tool. Usage of libprotoident itself is through functions beginning with 'lpi_'. -The libprotoident API functions themselves are documented in +The libprotoident API functions themselves are documented in lib/libprotoident.h if you need further guidance. -Further documentation of the API can also be found at +Further documentation of the API can also be found at http://wand.net.nz/trac/libprotoident/wiki/DeveloperDocs -If all else fails, drop us a line at contact@wand.net.nz. +If all else fails, drop us a line at contact@wand.net.nz. From ae750483b74c19566a97501f39b79aa86026abd9 Mon Sep 17 00:00:00 2001 From: Shane Alcock Date: Wed, 28 Jun 2017 14:41:32 +1200 Subject: [PATCH 68/68] Add -B option to all tools to control reporter thread buffering By default, libtrace will buffer 100 results before signalling the reporter thread to start reading them. This means that our tools can appear to be inactive when monitoring relatively quiet live interfaces, when in reality they are just waiting for more flows to expire before firing up the reporter. I've set a default of 10 completed flows, so no tool should be inactive for too long in the presence of traffic. --- tools/arff/lpi_arff.cc | 12 ++++++++++-- tools/find_unknown/lpi_find_unknown.cc | 13 ++++++++++--- tools/protoident/lpi_protoident.cc | 12 ++++++++++-- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/tools/arff/lpi_arff.cc b/tools/arff/lpi_arff.cc index 0153277e..f95762e9 100644 --- a/tools/arff/lpi_arff.cc +++ b/tools/arff/lpi_arff.cc @@ -487,7 +487,7 @@ static void cleanup_signal(int sig) static void usage(char *prog) { printf("Usage details for %s\n\n", prog); - printf("%s [-l ] [-T] [-b] [-d ] [-f ] [-R] [-t ] inputURI [inputURI ...]\n\n", prog); + printf("%s [-l ] [-T] [-b] [-d ] [-f ] [-R] [-t ] [-B buflen] inputURI [inputURI ...]\n\n", prog); printf("Options:\n"); printf(" -l Determine direction based on representing the 'inside'\n"); printf(" portion of the network\n"); @@ -498,6 +498,7 @@ static void usage(char *prog) printf(" -f Ignore flows that do not match the given BPF filter\n"); printf(" -R Ignore flows involving private RFC 1918 address space\n"); printf(" -t Share the workload over the given number of threads\n"); + printf(" -B Buffer results until there are results waiting\n"); exit(0); } @@ -509,6 +510,7 @@ int main(int argc, char *argv[]) struct sigaction sigact; struct globalopts opts; int threads = 1; + int bufferresults = 10; bool opt_true = true; bool opt_false = false; @@ -539,7 +541,7 @@ int main(int argc, char *argv[]) trace_set_stopping_cb(reporter, stop_reporter); trace_set_result_cb(reporter, per_result); - while ((opt = getopt(argc, argv, "l:bd:f:RhTt:")) != EOF) { + while ((opt = getopt(argc, argv, "l:bB:d:f:RhTt:")) != EOF) { switch (opt) { case 'l': opts.local_mac = optarg; @@ -548,6 +550,11 @@ int main(int argc, char *argv[]) case 'b': opts.require_both = true; break; + case 'B': + bufferresults = atoi(optarg); + if (bufferresults <= 0) + bufferresults = 1; + break; case 'd': dir = atoi(optarg); if (dir == 0) @@ -655,6 +662,7 @@ int main(int argc, char *argv[]) trace_set_perpkt_threads(currenttrace, threads); + trace_set_reporter_thold(currenttrace, bufferresults); trace_set_combiner(currenttrace, &combiner_unordered, (libtrace_generic_t){0}); diff --git a/tools/find_unknown/lpi_find_unknown.cc b/tools/find_unknown/lpi_find_unknown.cc index 99df465a..ec38ce7d 100644 --- a/tools/find_unknown/lpi_find_unknown.cc +++ b/tools/find_unknown/lpi_find_unknown.cc @@ -389,7 +389,7 @@ static void cleanup_signal(int sig) { static void usage(char *prog) { printf("Usage details for %s\n\n", prog); - printf("%s [-l ] [-T] [-b] [-d ] [-f ] [-R] [-H] [-t ] inputURI [inputURI ...]\n\n", prog); + printf("%s [-l ] [-T] [-b] [-d ] [-f ] [-R] [-H] [-t ] [-B Determine direction based on representing the 'inside' \n portion of the network\n"); printf(" -T Use trace direction tags to determine direction\n"); @@ -399,6 +399,7 @@ static void usage(char *prog) { printf(" -R Ignore flows involving private RFC 1918 address space\n"); printf(" -H Ignore flows that do not meet the criteria for an SPNAT hole\n"); printf(" -t Share the workload over the given number of threads\n"); + printf(" -B Buffer results until there are results waiting\n"); exit(0); } @@ -413,6 +414,7 @@ int main(int argc, char *argv[]) { char *filterstring = NULL; int dir; int threads = 1; + int bufferresults = 1; libtrace_callback_set_t *processing, *reporter; @@ -434,7 +436,7 @@ int main(int argc, char *argv[]) { trace_set_stopping_cb(reporter, stop_reporter); trace_set_result_cb(reporter, per_result); - while ((opt = getopt(argc, argv, "l:bHd:f:RhTt:")) != EOF) { + while ((opt = getopt(argc, argv, "l:bB:Hd:f:RhTt:")) != EOF) { switch (opt) { case 'l': opts.local_mac = optarg; @@ -443,6 +445,11 @@ int main(int argc, char *argv[]) { case 'b': opts.require_both = true; break; + case 'B': + bufferresults = atoi(optarg); + if (bufferresults <= 0) + bufferresults = 1; + break; case 'd': dir = atoi(optarg); if (dir == 0) @@ -524,7 +531,7 @@ int main(int argc, char *argv[]) { } trace_set_perpkt_threads(currenttrace, threads); - + trace_set_reporter_thold(currenttrace, bufferresults); trace_set_combiner(currenttrace, &combiner_unordered, (libtrace_generic_t){0}); diff --git a/tools/protoident/lpi_protoident.cc b/tools/protoident/lpi_protoident.cc index 2bae06cd..8d0c7618 100644 --- a/tools/protoident/lpi_protoident.cc +++ b/tools/protoident/lpi_protoident.cc @@ -383,7 +383,7 @@ static void cleanup_signal(int sig) { static void usage(char *prog) { printf("Usage details for %s\n\n", prog); - printf("%s [-l ] [-T] [-b] [-d ] [-f ] [-R] [-H] [-t ] inputURI [inputURI ...]\n\n", prog); + printf("%s [-l ] [-T] [-b] [-d ] [-f ] [-R] [-H] [-t ] [-B buflen] inputURI [inputURI ...]\n\n", prog); printf("Options:\n"); printf(" -l Determine direction based on representing the 'inside' \n portion of the network\n"); printf(" -T Use trace direction tags to determine direction\n"); @@ -393,6 +393,7 @@ static void usage(char *prog) { printf(" -R Ignore flows involving private RFC 1918 address space\n"); printf(" -H Ignore flows that do not meet the criteria for an SPNAT hole\n"); printf(" -t Share the workload over the given number of threads\n"); + printf(" -B Buffer results until there are results waiting\n"); exit(0); } @@ -407,6 +408,7 @@ int main(int argc, char *argv[]) { char *filterstring = NULL; int dir; int threads = 1; + int bufferresults = 10; libtrace_callback_set_t *processing, *reporter; @@ -428,7 +430,7 @@ int main(int argc, char *argv[]) { trace_set_stopping_cb(reporter, stop_reporter); trace_set_result_cb(reporter, per_result); - while ((opt = getopt(argc, argv, "l:bHd:f:RhTt:")) != EOF) { + while ((opt = getopt(argc, argv, "l:bB:Hd:f:RhTt:")) != EOF) { switch (opt) { case 'l': opts.local_mac = optarg; @@ -437,6 +439,11 @@ int main(int argc, char *argv[]) { case 'b': opts.require_both = true; break; + case 'B': + bufferresults = atoi(optarg); + if (bufferresults <= 0) + bufferresults = 1; + break; case 'd': dir = atoi(optarg); if (dir == 0) @@ -518,6 +525,7 @@ int main(int argc, char *argv[]) { } trace_set_perpkt_threads(currenttrace, threads); + trace_set_reporter_thold(currenttrace, bufferresults); trace_set_combiner(currenttrace, &combiner_unordered, (libtrace_generic_t){0});