Merge branch 'develop'

README

@@ -1,7 +1,7 @@
-libprotoident 2.0.10
+libprotoident 2.0.11
 
 ---------------------------------------------------------------------------
-Copyright (c) 2011-2016 The University of Waikato, Hamilton, New Zealand.
+Copyright (c) 2011-2017 The University of Waikato, Hamilton, New Zealand.
 All rights reserved.
 
 This code has been developed by the University of Waikato WAND
@@ -24,6 +24,7 @@ With contributions from:
     Aaron Murrihy
     Paweł Foremski <[email protected]>
     Fabian Weisshaar <[email protected]>
+    Jeroen Roovers
 
 Introduction
 ============
@@ -36,10 +37,10 @@ some tools that can be used to perform simple analysis of traffic flows.
 
 Required Libraries
 ==================
-libtrace
+libtrace 4.0.1 or later
 	* available from http://research.wand.net.nz/software/libtrace.php
 
-libflowmanager 2.0.4 or later
+libflowmanager 3.0.0 or later
 	* optional, but required to build the tools
 	* available from http://research.wand.net.nz/software/libflowmanager.php
 
@@ -208,7 +209,7 @@ There are currently four tools included with libprotoident.
          * Flow start time (as a Unix timestamp)
 
  * lpi_live   (DEPRECATED)
-  
+
    Description:
    	This tool reports byte and packet counts (both inbound and outbound)
 	for each identified protocol in real-time. Identification of a flow
@@ -261,7 +262,7 @@ There are currently four tools included with libprotoident.
 		- out_curr_flows = outbound flows active at the period end
 	* The application protocol being measured
 	* The value for the measured statistic
-		
+
 API
 ===
 
@@ -270,15 +271,15 @@ use the libprotoident API. The API is very simple and the best way to learn it
 is to examine how the existing tools work. The source for the tools is
 located in the tools/ directory.
 
-The tools use libflowmanager to do the flow tracking, using functions beginning
-with 'lfm_'. You will probably want to incorporate this into your own tool.
-Usage of libprotoident itself is through functions beginning with 'lpi_'.
+The tools use libflowmanager to do the flow tracking, using an instance of a
+FlowManager class. You will probably want to incorporate this into your own
+tool. Usage of libprotoident itself is through functions beginning with 'lpi_'.
 
-The libprotoident API functions themselves are documented in 
+The libprotoident API functions themselves are documented in
 lib/libprotoident.h if you need further guidance.
 
-Further documentation of the API can also be found at 
+Further documentation of the API can also be found at
 http://wand.net.nz/trac/libprotoident/wiki/DeveloperDocs
 
-If all else fails, drop us a line at [email protected].	
+If all else fails, drop us a line at [email protected].
 

configure.ac

@@ -1,4 +1,4 @@
-AC_INIT(libprotoident, 2.0.10, [email protected])
+AC_INIT(libprotoident, 2.0.11, [email protected])
 
 AM_INIT_AUTOMAKE
 AC_CONFIG_SRCDIR(lib/libprotoident.cc)
@@ -50,7 +50,7 @@ AC_ARG_WITH(tools,
 AC_CHECK_LIB([trace], [trace_get_payload_length],,trace_found=0)
 
 if test "$build_tools" = yes; then
-	AC_CHECK_LIB([flowmanager], [lfm_release_flow],,lfm_found=0)
+	AC_CHECK_LIB([flowmanager], [lfm_version_three],,lfm_found=0)
 fi
 
 AC_CHECK_HEADERS([arpa/inet.h fcntl.h inttypes.h netdb.h netinet/in.h stdint.h stdlib.h string.h sys/socket.h sys/time.h syslog.h unistd.h])

lib/Makefile.am

@@ -10,4 +10,5 @@ libprotoident_la_SOURCES=libprotoident.h libprotoident.cc \
 INCLUDES=@ADD_INCLS@
 libprotoident_la_LIBADD = @ADD_LIBS@ tcp/libprotoident_tcp.la \
 	udp/libprotoident_udp.la
-libprotoident_la_LDFLAGS = @ADD_LDFLAGS@ -version-info 2:10:0
+libprotoident_la_LDFLAGS = @ADD_LDFLAGS@ -version-info 2:11:0
+libprotoident_la_CPPFLAGS = -Werror

lib/libprotoident.h

@@ -316,6 +316,40 @@ typedef enum {
         LPI_PROTO_ZOOM,
         LPI_PROTO_S7COMM,
         LPI_PROTO_MAXICLOUD,
+        LPI_PROTO_GLUPTEBA,
+        LPI_PROTO_WNS,
+        LPI_PROTO_PANDATV,
+        LPI_PROTO_FACEBOOK_TURN,
+        LPI_PROTO_DESTINY,
+        LPI_PROTO_QCLOUD_ILVB,
+        LPI_PROTO_BITCOIN,
+        LPI_PROTO_LIFEFORGE,
+        LPI_PROTO_ACESTREAM,
+        LPI_PROTO_MAPLESTORY_CHINA,
+        LPI_PROTO_NDT_TPUT,
+        LPI_PROTO_RELAY,
+        LPI_PROTO_DOUYU,
+        LPI_PROTO_IDRIVE_SYNC,
+        LPI_PROTO_TWITCASTING,
+        LPI_PROTO_THE_DIVISION,
+        LPI_PROTO_BLACKDESERT,
+        LPI_PROTO_REALVNC,
+        LPI_PROTO_DOGECOIN,
+        LPI_PROTO_FUCKCOIN,
+        LPI_PROTO_OURWORLD,
+        LPI_PROTO_GRAAL_ONLINE_ERA,
+        LPI_PROTO_APPEAR_IN,
+        LPI_PROTO_VAINGLORY,
+        LPI_PROTO_WEIQI,
+        LPI_PROTO_4D,
+        LPI_PROTO_TANKIX,
+        LPI_PROTO_IPSHARKK,
+        LPI_PROTO_NET_MFP,
+        LPI_PROTO_SPEEDIN,
+        LPI_PROTO_CROSSFIRE,
+        LPI_PROTO_DASH,
+        LPI_PROTO_AIRMEDIA,
+        LPI_PROTO_GIOP,
 
         /* UDP Protocols */
         LPI_PROTO_UDP,
@@ -327,6 +361,7 @@ typedef enum {
         LPI_PROTO_UDP_QUAKE,
         LPI_PROTO_UDP_STEAM,
         LPI_PROTO_UDP_STEAM_FRIENDS,
+        LPI_PROTO_UDP_STEAM_INHOMEBROADCAST,
         LPI_PROTO_UDP_WIN_MESSAGE,
         LPI_PROTO_UDP_GAMESPY,
         LPI_PROTO_UDP_EMULE,
@@ -485,6 +520,35 @@ typedef enum {
         LPI_PROTO_UDP_YY,
         LPI_PROTO_UDP_OVERWATCH,
         LPI_PROTO_UDP_BACNET,
+        LPI_PROTO_UDP_ARK_SURVIVAL,
+        LPI_PROTO_UDP_360P2P,
+        LPI_PROTO_UDP_PORTMAP_RPC,
+        LPI_PROTO_UDP_NINTENDO,
+        LPI_PROTO_UDP_CHIVALRY,
+        LPI_PROTO_UDP_DOYO,
+        LPI_PROTO_UDP_NETCAT_CCTV,
+        LPI_PROTO_UDP_N2PING,
+        LPI_PROTO_UDP_RAMSEY_DASH,
+        LPI_PROTO_UDP_UBISOFT_GAMES,
+        LPI_PROTO_UDP_THE_CREW,
+        LPI_PROTO_UDP_TURBOVPN,
+        LPI_PROTO_UDP_GEARSOFWAR,
+        LPI_PROTO_UDP_RDP,
+        LPI_PROTO_UDP_HOTS,
+        LPI_PROTO_UDP_VPNMASTER,
+        LPI_PROTO_UDP_DIANSHIJIA,
+        LPI_PROTO_UDP_PS4_REMOTEPLAY,
+        LPI_PROTO_UDP_STARCITIZEN,
+        LPI_PROTO_UDP_WEBEX,
+        LPI_PROTO_UDP_HALO_ONLINE,
+        LPI_PROTO_UDP_GOTOMEETING,
+        LPI_PROTO_UDP_CROSSOUT,
+        LPI_PROTO_UDP_UMEYE,
+        LPI_PROTO_UDP_RISING_STORM,
+        LPI_PROTO_UDP_CROSSFIRE,
+        LPI_PROTO_UDP_MERAKICLOUD,
+        LPI_PROTO_UDP_SNAPVPN,
+        LPI_PROTO_UDP_DAHUA,
 
 	/* Patterns that we can match, but do not know the protocol */
 	LPI_PROTO_REJECTION,	/* All responses are 0x02 */

lib/proto_common.cc

@@ -305,6 +305,22 @@ bool match_file_header(uint32_t payload) {
         if (MATCH(payload, 0x00, 0x00, 0x00, 0x20))
                 return true;
 
+        /* TIFF */
+        if (MATCH(payload, 0x49, 0x49, 0x2a, 0x00))
+                return true;
+
+        /* LZMA */
+        if (MATCH(payload, 0x5d, 0x00, 0x00, 0x80))
+                return true;
+
+        /* Source engine BSP file */
+        if (MATCH(payload, 'V', 'B', 'S', 'P'))
+                return true;
+
+        /* Old coralreef trace files! */
+        if (MATCHSTR(payload, "\xff\xff\x44\x00"))
+                return true;
+
         /* I'm pretty sure the following are files of some type or another.
          * They crop up pretty often in our test data sets, so I'm going to
          * put them in here.
@@ -807,3 +823,24 @@ bool match_yy_payload(uint32_t payload, uint32_t len) {
         return false;
 }
 
+
+/* Byte swapping functions for various inttypes */
+uint64_t byteswap64(uint64_t num)
+{
+        return (byteswap32((num&0xFFFFFFFF00000000ULL)>>32))
+              |((uint64_t)byteswap32(num&0x00000000FFFFFFFFULL)<<32);
+}
+
+uint32_t byteswap32(uint32_t num)
+{
+        return ((num&0x000000FFU)<<24)
+                | ((num&0x0000FF00U)<<8)
+                | ((num&0x00FF0000U)>>8)
+                | ((num&0xFF000000U)>>24);
+}
+
+uint16_t byteswap16(uint16_t num)
+{
+        return ((num<<8)&0xFF00)|((num>>8)&0x00FF);
+}
+

lib/proto_common.h

@@ -57,6 +57,69 @@
         (memcmp(&(x),(st),sizeof(x))==0)
 
 
+/** Byteswaps a 64-bit value.
+ *
+ * @param num           The value to be byteswapped.
+ * @return The byteswapped 64-bit number
+ *
+ */
+uint64_t byteswap64(uint64_t num);
+
+/** Byteswaps a 32-bit value.
+ *
+ * @param num           The value to be byteswapped.
+ * @return The byteswapped 32-bit number
+ *
+ */
+uint32_t byteswap32(uint32_t num);
+
+/** Byteswaps a 16-bit value.
+ *
+ * @param num           The value to be byteswapped.
+ * @return The byteswapped 16-bit number
+ *
+ */
+uint16_t byteswap16(uint16_t num);
+
+
+#if __BYTE_ORDER == __BIG_ENDIAN
+#define bswap_host_to_be64(num) ((uint64_t)(num))
+#define bswap_host_to_le64(num) byteswap64(num)
+#define bswap_host_to_be32(num) ((uint32_t)(num))
+#define bswap_host_to_le32(num) byteswap32(num)
+#define bswap_host_to_be16(num) ((uint16_t)(num))
+#define bswap_host_to_le16(num) byteswap16(num)
+
+#define bswap_be_to_host64(num) ((uint64_t)(num))
+#define bswap_le_to_host64(num) byteswap64(num)
+#define bswap_be_to_host32(num) ((uint32_t)(num))
+#define bswap_le_to_host32(num) byteswap32(num)
+#define bswap_be_to_host16(num) ((uint16_t)(num))
+#define bswap_le_to_host16(num) byteswap16(num)
+
+/* We use ntoh*() here, because the compiler may
+ * attempt to optimise it
+  */
+#elif __BYTE_ORDER == __LITTLE_ENDIAN
+#define bswap_host_to_be64(num) (byteswap64(num))
+#define bswap_host_to_le64(num) ((uint64_t)(num))
+#define bswap_host_to_be32(num) (htonl(num))
+#define bswap_host_to_le32(num) ((uint32_t)(num))
+#define bswap_host_to_be16(num) (htons(num))
+#define bswap_host_to_le16(num) ((uint16_t)(num))
+
+#define bswap_be_to_host64(num) (byteswap64(num))
+#define bswap_le_to_host64(num) ((uint64_t)(num))
+#define bswap_be_to_host32(num) (ntohl(num))
+#define bswap_le_to_host32(num) ((uint32_t)(num))
+#define bswap_be_to_host16(num) (ntohs(num))
+#define bswap_le_to_host16(num) ((uint16_t)(num))
+
+#else
+#error "Unknown byte order"
+#endif
+
+
 bool match_str_either(lpi_data_t *data, const char *string);
 bool match_str_both(lpi_data_t *data, const char *string1,
         const char *string2);