Merge pull request #237 from sfu-cl-lab/master

merge Mingyi's version to the master

code/factorbase/src/main/java/ca/sfu/cs/factorbase/app/RunBB.java

@@ -52,12 +52,14 @@ public static void main(
         RuntimeLogger.logRunTime(logger, "Logger + Config Initialization", start, configEnd);
 
         long databaseStart = System.currentTimeMillis();
+        String databaseCollation = config.getProperty("dbcollation");
         FactorBaseDataBase factorBaseDatabase = new MySQLFactorBaseDataBase(
             new FactorBaseDataBaseInfo(config),
             config.getProperty("dbaddress"),
             config.getProperty("dbname"),
             config.getProperty("dbusername"),
             config.getProperty("dbpassword"),
+            databaseCollation,
             countingStrategy
         );
         RuntimeLogger.logRunTime(logger, "Creating Database Connection", databaseStart, System.currentTimeMillis());
@@ -92,6 +94,7 @@ public static void main(
         BayesBaseH.runBBH(
             factorBaseDatabase,
             globalLattice,
+            databaseCollation,
             countingStrategy
         );
 

code/factorbase/src/main/java/ca/sfu/cs/factorbase/database/MySQLFactorBaseDataBase.java

@@ -46,6 +46,7 @@ public class MySQLFactorBaseDataBase implements FactorBaseDataBase {
     private Connection dbConnection;
     private FactorBaseDataBaseInfo dbInfo;
     private Map<String, DataExtractor> dataExtractors;
+    private String dbCollation;
     private CountingStrategy countingStrategy;
 
 
@@ -66,10 +67,12 @@ public MySQLFactorBaseDataBase(
         String dbname,
         String username,
         String password,
+        String dbCollation,
         CountingStrategy countingStrategy
     ) throws DataBaseException {
         this.dbInfo = dbInfo;
         this.baseDatabaseName = dbname;
+        this.dbCollation = dbCollation;
         this.countingStrategy = countingStrategy;
         String baseConnectionString = MessageFormat.format(CONNECTION_STRING, dbaddress, dbname);
         Properties connectionProperties = getConnectionStringProperties(username, password);
@@ -89,20 +92,23 @@ public void setupDatabase() throws DataBaseException {
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "initialize_databases.sql",
-                this.baseDatabaseName
+                this.baseDatabaseName,
+                this.dbCollation
             );
 
             // Switch to start using the setup database.
             this.dbConnection.setCatalog(this.dbInfo.getSetupDatabaseName());
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "metadata.sql",
-                this.baseDatabaseName
+                this.baseDatabaseName,
+                this.dbCollation
             );
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "metadata_storedprocedures.sql",
                 this.baseDatabaseName,
+                this.dbCollation,
                 "//"
             );
             MySQLScriptRunner.callSP(this.dbConnection, "find_values");
@@ -111,7 +117,8 @@ public void setupDatabase() throws DataBaseException {
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "latticegenerator_initialize.sql",
-                this.baseDatabaseName
+                this.baseDatabaseName,
+                this.dbCollation
             );
 
             // Switch to start using the BN database.
@@ -120,50 +127,59 @@ public void setupDatabase() throws DataBaseException {
             RuntimeLogger.setupLoggingTable(
                 this.dbConnection,
                 this.baseDatabaseName,
-                this.dbInfo.getBNDatabaseName()
+                this.dbInfo.getBNDatabaseName(),
+                this.dbCollation
             );
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "latticegenerator_initialize_local.sql",
-                this.baseDatabaseName
+                this.baseDatabaseName,
+                this.dbCollation
             );
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "latticegenerator_populate.sql",
                 this.baseDatabaseName,
+                this.dbCollation,
                 "//"
             );
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "transfer_initialize.sql",
-                this.baseDatabaseName
+                this.baseDatabaseName,
+                this.dbCollation
             );
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "transfer_cascade.sql",
                 this.baseDatabaseName,
+                this.dbCollation,
                 "//"
             );
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "modelmanager_initialize.sql",
-                this.baseDatabaseName
+                this.baseDatabaseName,
+                this.dbCollation
             );
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "metaqueries_initialize.sql",
-                this.baseDatabaseName
+                this.baseDatabaseName,
+                this.dbCollation
             );
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "metaqueries_populate.sql",
                 this.baseDatabaseName,
+                this.dbCollation,
                 "//"
             );
             MySQLScriptRunner.runScript(
                 this.dbConnection,
                 Config.SCRIPTS_DIRECTORY + "metaqueries_RChain.sql",
                 this.baseDatabaseName,
+                this.dbCollation,
                 "//"
             );
         } catch (SQLException | IOException e) {

code/factorbase/src/main/java/ca/sfu/cs/factorbase/learning/BayesBaseH.java

@@ -108,6 +108,7 @@ public class BayesBaseH {
     public static void runBBH(
         FactorBaseDataBase database,
         RelationshipLattice globalLattice,
+        String databaseCollation,
         CountingStrategy countingStrategy
     ) throws SQLException, IOException, DataBaseException, DataExtractionException, ParsingException, ScoringException {
         initProgram(FirstRunning);
@@ -124,6 +125,7 @@ public static void runBBH(
         StructureLearning(
             database,
             con2,
+            databaseCollation,
             countingStrategy,
             globalLattice
         );
@@ -206,6 +208,7 @@ public static void runBBH(
     private static void StructureLearning(
         FactorBaseDataBase database,
         Connection conn,
+        String databaseCollation,
         CountingStrategy countingStrategy,
         RelationshipLattice lattice
     ) throws SQLException, IOException, DataBaseException, DataExtractionException, ParsingException, ScoringException {
@@ -221,7 +224,8 @@ private static void StructureLearning(
         MySQLScriptRunner.runScript(
             conn,
             Config.SCRIPTS_DIRECTORY + "modelmanager_populate.sql",
-            databaseName
+            databaseName,
+            databaseCollation
         );
 
         // Handle rnodes in a bottom-up way following the lattice.

code/factorbase/src/main/java/ca/sfu/cs/factorbase/learning/CountsManager.java

@@ -69,6 +69,7 @@ public class CountsManager {
     private static boolean generatePDPInfo;
     private static String linkCorrelation;
     private static long dbTemporaryTableSize;
+    private static String dbCollation;
     /*
      * cont is Continuous
      * ToDo: Refactor
@@ -93,7 +94,7 @@ public static void buildCT(
         RuntimeLogger.addLogEntry(dbConnection);
         try (Statement statement = dbConnection.createStatement()) {
             statement.execute("DROP SCHEMA IF EXISTS " + dbInfo.getCTDatabaseName() + ";");
-            statement.execute("CREATE SCHEMA " + dbInfo.getCTDatabaseName() + " /*M!100316 COLLATE utf8_general_ci*/;");
+            statement.execute("CREATE SCHEMA " + dbInfo.getCTDatabaseName() + " COLLATE " + dbCollation + ";");
         }
 
         // Propagate metadata based on the FunctorSet.
@@ -492,6 +493,7 @@ private static void setVarsFromConfig() {
         dbPassword = conf.getProperty("dbpassword");
         dbaddress = conf.getProperty("dbaddress");
         dbTemporaryTableSize = Math.round(1024 * 1024 * 1024 * Double.valueOf(conf.getProperty("dbtemporarytablesize")));
+        dbCollation = conf.getProperty("dbcollation");
         linkCorrelation = conf.getProperty("LinkCorrelations");
         cont = conf.getProperty("Continuous");
         String loggingLevel = conf.getProperty("LoggingLevel");

code/factorbase/src/main/java/ca/sfu/cs/factorbase/util/MySQLScriptRunner.java

@@ -32,9 +32,14 @@ private MySQLScriptRunner() {
      *
      * @param fileName - the file to create a copy of with the variables filled in.
      * @param databaseName - the name of the database to replace instances of "@database@" with.
+     * @param databaseCollation - the collation to use for the tables created.
      * @throws IOException if there is an issue reading from the script.
      */
-    private static String prepareFile(String fileName, String databaseName) throws IOException {
+    private static String prepareFile(
+        String fileName,
+        String databaseName,
+        String databaseCollation
+    ) throws IOException {
         InputStream inputStream = MySQLScriptRunner.class.getClassLoader().getResourceAsStream(fileName);
         if (inputStream == null) {
             throw new FileNotFoundException("Unable to read the file: " + fileName);
@@ -59,6 +64,7 @@ private static String prepareFile(String fileName, String databaseName) throws I
         String finalOutput = "";
         while (line != null) {
             line = line.replace("@database@", databaseName);
+            line = line.replace("@dbcollation@", databaseCollation);
             finalOutput += line + System.getProperty("line.separator");
             line = input.readLine();
         }
@@ -94,11 +100,17 @@ public static void callSP(Connection dbConnection, String spName) throws SQLExce
      * @param dbConnection - connection to the database to execute the script on.
      * @param scriptFileName - the path to the MySQL script to execute.
      * @param databaseName - the name of the database to replace instances of "@database@" with.
+     * @param databaseCollation - the collation to use for the tables created.
      * @throws SQLException if there is an issue executing the command(s).
      * @throws IOException if there is an issue reading from the script.
      */
-    public static void runScript(Connection dbConnection, String scriptFileName, String databaseName) throws SQLException, IOException {
-        runScript(dbConnection, scriptFileName, databaseName, ";");
+    public static void runScript(
+        Connection dbConnection,
+        String scriptFileName,
+        String databaseName,
+        String databaseCollation
+    ) throws SQLException, IOException {
+        runScript(dbConnection, scriptFileName, databaseName, databaseCollation, ";");
     }
 
 
@@ -110,12 +122,19 @@ public static void runScript(Connection dbConnection, String scriptFileName, Str
      * @param dbConnection - connection to the database to execute the script on.
      * @param scriptFileName - the path to the MySQL script to execute.
      * @param databaseName - the name of the database to replace instances of "@database@" with.
+     * @param databaseCollation - the collation to use for the tables created.
      * @param delimiter - the delimiter to use when reading the commands from the given script.
      * @throws SQLException if there is an issue executing the command(s).
      * @throws IOException if there is an issue reading from the script.
      */
-    public static void runScript(Connection dbConnection, String scriptFileName, String databaseName, String delimiter) throws SQLException, IOException {
-        String newScriptFileName = prepareFile(scriptFileName, databaseName);
+    public static void runScript(
+        Connection dbConnection,
+        String scriptFileName,
+        String databaseName,
+        String databaseCollation,
+        String delimiter
+    ) throws SQLException, IOException {
+        String newScriptFileName = prepareFile(scriptFileName, databaseName, databaseCollation);
 
         ScriptRunner runner = new ScriptRunner(dbConnection);
         try (

code/factorbase/src/main/java/ca/sfu/cs/factorbase/util/RuntimeLogger.java

@@ -94,13 +94,15 @@ public static void logExecutedQuery(Logger logger, String query) {
     public static void setupLoggingTable(
         Connection dbConnection,
         String baseDatabaseName,
-        String loggingTableDatabaseName
+        String loggingTableDatabaseName,
+        String databaseCollation
     ) throws SQLException, IOException {
         dbName = loggingTableDatabaseName;
         MySQLScriptRunner.runScript(
             dbConnection,
             Config.SCRIPTS_DIRECTORY + "logging.sql",
-            baseDatabaseName
+            baseDatabaseName,
+            databaseCollation
         );
     }
 

code/factorbase/src/main/resources/scripts/initialize_databases.sql

@@ -1,5 +1,5 @@
 -- Initialize the databases required by FactorBase.
-/*M!100316 SET collation_server = 'utf8_general_ci';*/
+
 SET collation_server = 'utf8_general_ci';
 
 DROP SCHEMA IF EXISTS @database@_setup;

config.cfg

@@ -1,9 +1,10 @@
 # Database Configurations
-dbaddress = mysql://127.0.0.1 
-dbname = name_of_database
+dbaddress = mysql://127.0.0.1
+dbname = name_of_databases
 dbusername = database_username
 dbpassword = database_password
 dbtemporarytablesize = 4
+dbcollation = latin1_swedish_ci
 
 # FactorBase Configurations
 AutomaticSetup = 1

make_xml_sum_to_one.py

@@ -0,0 +1,60 @@
+"""
+This script takes an xml_file and makes all the probabilities sum to 1.
+You need to install bs4 library first (pip install bs4)
+
+"""
+
+
+from bs4 import BeautifulSoup
+
+# Reading the data inside the XML file to a variable
+xml_file = '/Users/parmis/Desktop/Bif_functor_road_scene_2.xml' # change the path to your xml path
+
+with open(xml_file, 'r') as f:
+    data = f.read()
+
+# Parsing the XML using BeautifulSoup
+Bs_data = BeautifulSoup(data, "xml")
+
+# Finding all instances of the 'VARIABLE' tag
+all_vars = Bs_data.find_all('VARIABLE')
+
+
+
+# Counting all the possible values for a variable
+
+print("Counting all the possible values for a variable")
+var_to_num_values = {}
+for var in all_vars:
+    name = var.find('NAME')
+    outcome_elements = var.find_all('OUTCOME')
+    var_to_num_values[name.text] = len(outcome_elements)
+
+
+
+
+all_defs = Bs_data.find_all('DEFINITION')
+
+
+
+print("Making all the conditional probablities sum to 1")
+for defin in all_defs:
+    name = defin.find('FOR').text
+    count_possible_values = var_to_num_values[name]
+
+    table = defin.find('TABLE').get_text().split()
+    table = list(map(float, table))
+    new_table = []
+    for i in range(0,len(table), count_possible_values):
+        current_values = table[i: i + count_possible_values ]
+        summ = sum(current_values)
+        for i in range(len(current_values)):
+            current_values[i] =  current_values[i] / summ
+        new_table.extend(current_values)
+
+    defin.find('TABLE').string = ' '.join([str(item) for item in new_table])
+
+# Save the updated XML
+with open(xml_file, 'w') as f:
+    f.write(Bs_data.prettify())
+print("File is updated!")

travis-resources/config.cfg

@@ -4,6 +4,7 @@ dbname = unielwin
 dbusername = root
 dbpassword = 123456
 dbtemporarytablesize = 4
+dbcollation = latin1_swedish_ci
 
 # FactorBase Configurations
 AutomaticSetup = 1