[Demo] Example protobuf based packet processor

WowPacketParser/App.config

@@ -201,6 +201,8 @@
         <add key="hotfix_optional_data"                   value="false"/>
         <add key="hotfixes"                               value="false"/>
 
+        <add key="creature_movement"                      value="false"/>
+
 
         <!--
                  Option:      ParseAllHotfixes

WowPacketParser/Enums/SQLOutput.cs

@@ -19,6 +19,7 @@ public enum SQLOutput
         creature_addon,
         creature_equip_template,
         creature_model_info,
+        creature_movement,
         creature_queststarter,
         creature_questender,
         creature_template,

WowPacketParser/Loading/SniffFile.cs

@@ -83,18 +83,19 @@ private string FileName
             }
         }
 
-        public void ProcessFile()
+        public Packets ProcessFile()
         {
 
             try
             {
-                ProcessFileImpl();
+                return ProcessFileImpl();
             }
             catch (Exception ex)
             {
                 Trace.WriteLine(_logPrefix + " " + ex.GetType());
                 Trace.WriteLine(_logPrefix + " " + ex.Message);
                 Trace.WriteLine(_logPrefix + " " + ex.StackTrace);
+                return new Packets();
             }
             finally
             {
@@ -106,7 +107,7 @@ public void ProcessFile()
             }
         }
 
-        private void ProcessFileImpl()
+        private Packets ProcessFileImpl()
         {
             if (_compression != FileCompression.None)
                 _tempName = Decompress();
@@ -184,9 +185,9 @@ private void ProcessFileImpl()
 
                     var written = false;
 
+                    Packets packets = new() { Version = StructureVersion.ProtobufStructureVersion, DumpType = (uint)Settings.DumpFormat };
                     using (var writer = (Settings.DumpFormatWithTextToFile() ? new StreamWriter(outFileName, true) : null))
                     {
-                        Packets packets = new() { Version = StructureVersion.ProtobufStructureVersion, DumpType = (uint)Settings.DumpFormat };
                         var firstRead = true;
                         var firstWrite = true;
 
@@ -285,7 +286,7 @@ private void ProcessFileImpl()
                             // Close Writer, Stream - Dispose
                             packet.ClosePacket();
 
-                            if (_dumpFormat.IsUniversalProtobufType())
+                            if (_dumpFormat.IsUniversalProtobufType() || Settings.SQLOutputFlag != 0 || HotfixSettings.Instance.ShouldLog())
                                 packets.Packets_.Add(packet.Holder);
                         }, threadCount);
 
@@ -316,14 +317,14 @@ private void ProcessFileImpl()
                     Trace.WriteLine($"{_logPrefix}: {_stats}");
 
                     if (Settings.SQLOutputFlag != 0 || HotfixSettings.Instance.ShouldLog())
-                        WriteSQLs();
+                        WriteSQLs(packets);
 
                     if (Settings.LogPacketErrors)
                         WritePacketErrors();
 
                     GC.Collect(); // Force a GC collect after parsing a file. It seems to help.
 
-                    break;
+                    return packets;
                 }
                 case DumpFormatType.Pkt:
                 {
@@ -478,6 +479,8 @@ private void ProcessFileImpl()
                     break;
                 }
             }
+
+            return new Packets();
         }
 
         public static string GetHeader(string fileName)
@@ -560,14 +563,14 @@ private void BinaryDump(string fileName, ICollection<Packet> packets)
             BinaryPacketWriter.Write(SniffType.Pkt, fileName, Encoding.ASCII, packets);
         }
 
-        private void WriteSQLs()
+        private void WriteSQLs(Packets packets)
         {
             var sqlFileName = string.IsNullOrWhiteSpace(Settings.SQLFileName) ? $"{Utilities.FormattedDateTimeForFiles()}_{Path.GetFileName(FileName)}.sql" : Settings.SQLFileName;
 
             if (!string.IsNullOrWhiteSpace(Settings.SQLFileName))
                 return;
 
-            Builder.DumpSQL($"{_logPrefix}: Dumping sql", sqlFileName, GetHeader(FileName));
+            Builder.DumpSQL(new []{packets}, $"{_logPrefix}: Dumping sql", sqlFileName, GetHeader(FileName));
             Storage.ClearContainers();
         }
 

WowPacketParser/Misc/Extensions.cs

@@ -56,6 +56,24 @@ public static bool HasAnyFlagBit(this IConvertible value, IConvertible bit)
             return (uThis & uFlag) != 0;
         }
 
+        /// <summary>
+        /// Returns true if bit is set in value (&)
+        /// </summary>
+        /// <param name="value">An enum, int, ...</param>
+        /// <param name="bit">An int</param>
+        /// <returns>A boolean</returns>
+        public static bool HasAnyFlagBit(this UInt128 value, IConvertible bit) // uint128 doesnt implement IConvertible
+        {
+            var uBit = bit.ToInt32(null);
+
+            Contract.Assert(uBit >= 0 && uBit <= 127);
+
+            var uFlag = ((UInt128)1) << uBit;
+            var uThis = value;
+
+            return (uThis & uFlag) != 0;
+        }
+
         /// <summary>
         /// Return true if our string is a substring of any filter (case insensitive)
         /// </summary>

WowPacketParser/Misc/Settings.cs

@@ -19,7 +19,7 @@ public static class Settings
         public static readonly TargetedDatabase TargetedDatabase = Conf.GetEnum("TargetedDatabase", TargetedDatabase.WrathOfTheLichKing);
         public static readonly TargetedProject TargetedProject = Conf.GetEnum("TargetedProject", TargetedProject.TrinityCore);
         public static readonly DumpFormatType DumpFormat = Conf.GetEnum("DumpFormat", DumpFormatType.Text);
-        public static readonly ulong SQLOutputFlag = GetSQLOutputFlag();
+        public static readonly UInt128 SQLOutputFlag = GetSQLOutputFlag();
         public static readonly bool SQLOrderByKey = Conf.GetBoolean("SqlOrderByKey", false);
         public static readonly bool SaveTempSpawns = Conf.GetBoolean("SaveTempSpawns", false);
         public static readonly bool SaveExistingSpawns = Conf.GetBoolean("SaveExistingSpawns", false);
@@ -67,17 +67,17 @@ public static class Settings
         public static readonly bool UseDBC = Conf.GetBoolean("UseDBC", false);
         public static readonly bool ParseSpellInfos = Conf.GetBoolean("ParseSpellInfos", false);
 
-        private static ulong GetSQLOutputFlag()
+        private static UInt128 GetSQLOutputFlag()
         {
             var names = Enum.GetNames(typeof(SQLOutput));
             var values = Enum.GetValues(typeof(SQLOutput));
 
-            var result = 0ul;
+            UInt128 result = 0;
 
             for (var i = 0; i < names.Length; ++i)
             {
                 if (Conf.GetBoolean(names[i], false))
-                    result += (1ul << (int)values.GetValue(i));
+                    result += (((UInt128)1) << (int)values.GetValue(i));
             }
 
             return result;

WowPacketParser/PacketStructures/ProtoExtensions.cs

@@ -256,5 +256,116 @@
                 return CreateObjectType.Spawn;
             throw new ArgumentOutOfRangeException();
         }
+
+        public static ObjectType ToObjectType(this UniversalHighGuid highGuid)
+        {
+            switch (highGuid)
+            {
+                case UniversalHighGuid.Player:
+                    return ObjectType.Player;
+                case UniversalHighGuid.DynamicObject:
+                    return ObjectType.DynamicObject;
+                case UniversalHighGuid.Item:
+                    return ObjectType.Item;
+                case UniversalHighGuid.GameObject:
+                case UniversalHighGuid.Transport:
+                    return ObjectType.GameObject;
+                case UniversalHighGuid.Vehicle:
+                case UniversalHighGuid.Creature:
+                case UniversalHighGuid.Pet:
+                    return ObjectType.Unit;
+                case UniversalHighGuid.AreaTrigger:
+                    return ObjectType.AreaTrigger;
+                default:
+                    return ObjectType.Object;
+            }
+        }
+
+        public static string ToWowParserString(this UniversalGuid? guid)
+        {
+            if (guid == null)
+                return "0x0 (null)";
+            if (guid.KindCase == UniversalGuid.KindOneofCase.Guid128 && (guid.Guid128.High != 0 || guid.Guid128.Low != 0))
+            {
+                if (guid.HasEntry())
+                {
+                    // ReSharper disable once UseStringInterpolation
+                    return string.Format("Full: 0x{0:X16}{1:X16} {2}/{3} R{4}/S{5} Map: {6} Entry: {7} Low: {8}", guid.Guid128.High, guid.Guid128.Low,
+                        guid.Type, guid.Guid128.GetSubType(), guid.Guid128.GetRealmId(), guid.Guid128.GetServerId(), guid.Guid128.GetMapId(),
+                        guid.Entry, guid.GetLow());
+                }
+
+                // ReSharper disable once UseStringInterpolation
+                return string.Format("Full: 0x{0:X16}{1:X16} {2}/{3} R{4}/S{5} Map: {6} Low: {7}", guid.Guid128.High, guid.Guid128.Low,
+                    guid.Type, guid.Guid128.GetSubType(), guid.Guid128.GetRealmId(), guid.Guid128.GetServerId(), guid.Guid128.GetMapId(),
+                    guid.GetLow());
+            }
+            else if (guid.KindCase == UniversalGuid.KindOneofCase.Guid64 && (guid.Guid64.High != 0 || guid.Guid64.Low != 0))
+            {
+                if (guid.HasEntry())
+                {
+                    return string.Format("Full: 0x{0:X8} Type: {1} Entry: {2} Low: {0}", guid.GetLow(), guid.Type, guid.Entry);
+                }
+                return string.Format("Full: 0x{0:X8} Type: {1} Low: {0}", guid.GetLow(), guid.Type);
+            }
+
+            return "Full: 0x0";
+        }
+
+        public static uint GetLow(this UniversalGuid guid)
+        {
+            if (guid.KindCase == UniversalGuid.KindOneofCase.Guid128)
+                return (uint)(guid.Guid128.Low & 0xFFFFFFFFFF);
+            else if (guid.KindCase == UniversalGuid.KindOneofCase.Guid64)
+            {
+                switch (guid.Type)
+                {
+                    case UniversalHighGuid.Player:
+                    case UniversalHighGuid.DynamicObject:
+                    case UniversalHighGuid.RaidGroup:
+                    case UniversalHighGuid.Item:
+                        return (uint)(guid.Guid64.Low & 0x000FFFFFFFFFFFFF);
+                    case UniversalHighGuid.GameObject:
+                    case UniversalHighGuid.Transport:
+                    //case HighGuidType.MOTransport: ??
+                    case UniversalHighGuid.Vehicle:
+                    case UniversalHighGuid.Creature:
+                    case UniversalHighGuid.Pet:
+                        return (uint)(guid.Guid64.Low & 0x00000000FFFFFFFFul);
+                }
+
+                return (uint)(guid.Guid64.Low & 0x00000000FFFFFFFFul);
+            }
+
+            return 0;
+        }
+
+        public static uint GetEntry(this UniversalGuid128 guid)
+        {
+            return (uint)((guid.High >> 6) & 0x7FFFFF);
+        }
+
+        public static byte GetSubType(this UniversalGuid128 guid) => (byte)(guid.High & 0x3F);
+
+        public static ushort GetRealmId(this UniversalGuid128 guid) => (ushort)((guid.High >> 42) & 0x1FFF);
+
+        public static uint GetServerId(this UniversalGuid128 guid) => (uint)((guid.Low >> 40) & 0xFFFFFF);
+
+        public static ushort GetMapId(this UniversalGuid128 guid) => (ushort)((guid.High >> 29) & 0x1FFF);
+
+        public static bool HasEntry(this UniversalGuid guid)
+        {
+            switch (guid.Type)
+            {
+                case UniversalHighGuid.Creature:
+                case UniversalHighGuid.GameObject:
+                case UniversalHighGuid.Pet:
+                case UniversalHighGuid.Vehicle:
+                case UniversalHighGuid.AreaTrigger:
+                    return true;
+                default:
+                    return false;
+            }
+        }
     }
 }

WowPacketParser/Parsing/Proto/BaseProtoQueryBuilder.cs

@@ -0,0 +1,26 @@
+using System.Collections.Generic;
+using WowPacketParser.Proto;
+using WowPacketParser.Proto.Processing;
+
+namespace WowPacketParser.Parsing.Proto;
+
+public abstract class BaseProtoQueryBuilder : PacketProcessor<VoidType>, IProtoQueryBuilder
+{
+    public abstract bool IsEnabled();
+
+    public string Process(IReadOnlyList<Packets> packetsList)
+    {
+        foreach (var sniffFile in packetsList)
+        {
+            // more complex logic is possible, i.e. clear state after each file
+            foreach (var packet in sniffFile.Packets_)
+            {
+                Process(packet);
+            }
+        }
+
+        return GenerateQuery();
+    }
+
+    protected abstract string GenerateQuery();
+}

WowPacketParser/Parsing/Proto/IProtoQueryBuilder.cs

@@ -0,0 +1,10 @@
+using System.Collections.Generic;
+using WowPacketParser.Proto;
+
+namespace WowPacketParser.Parsing.Proto;
+
+public interface IProtoQueryBuilder
+{
+    bool IsEnabled();
+    string Process(IReadOnlyList<Packets> packets);
+}

WowPacketParser/Parsing/Proto/VoidType.cs

@@ -0,0 +1,3 @@
+namespace WowPacketParser.Parsing.Proto;
+
+public struct VoidType { }

WowPacketParser/Program.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Globalization;
 using System.IO;
@@ -7,6 +8,7 @@
 using WowPacketParser.Loading;
 using WowPacketParser.Misc;
 using WowPacketParser.Parsing.Parsers;
+using WowPacketParser.Proto;
 using WowPacketParser.SQL;
 
 namespace WowPacketParser
@@ -58,6 +60,8 @@ private static void Main(string[] args)
 
             SQLConnector.ReadDB();
 
+            List<Packets> parserPacketsList = new();
+
             var processStartTime = DateTime.Now;
             var count = 0;
             foreach (var file in files)
@@ -71,7 +75,7 @@ private static void Main(string[] args)
                 try
                 {
                     var sf = new SniffFile(file, Settings.DumpFormat, Tuple.Create(++count, files.Count));
-                    sf.ProcessFile();
+                    parserPacketsList.Add(sf.ProcessFile());
                 }
                 catch (IOException ex)
                 {
@@ -80,7 +84,7 @@ private static void Main(string[] args)
             }
 
             if (!string.IsNullOrWhiteSpace(Settings.SQLFileName) && Settings.DumpFormatWithSQL())
-                Builder.DumpSQL("Dumping global sql", Settings.SQLFileName, SniffFile.GetHeader("multi"));
+                Builder.DumpSQL(parserPacketsList, "Dumping global sql", Settings.SQLFileName, SniffFile.GetHeader("multi"));
 
             var processTime = DateTime.Now.Subtract(processStartTime);
             Trace.WriteLine($"Processing {files.Count} sniffs took { processTime.ToFormattedString() }.");