Samoaheron

README.md

@@ -65,6 +65,18 @@ cd incubator-samoa
 mvn -Papex package
 ```
 
+### Heron mode
+
+Simply clone the repository and install SAMOA.
+
+```bash
+git clone http://git.apache.org/incubator-samoa.git
+cd incubator-samoa
+mvn -Pheron package
+```
+
+The deployable jar for SAMOA will be in `target/SAMOA-Heron-0.5.0-SNAPSHOT.jar`.
+
 ### Local mode
 
 If you want to test SAMOA in a local environment, simply clone the repository and install SAMOA.

pom.xml

@@ -19,7 +19,8 @@
   #L%
   -->
 
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
     <name>Apache SAMOA</name>
@@ -102,6 +103,15 @@
                 <module>samoa-test</module>
             </modules>
         </profile>
+        <profile>
+            <id>heron</id>
+            <modules>
+                <module>samoa-instances</module>
+                <module>samoa-api</module>
+                <module>samoa-heron</module>
+                <module>samoa-test</module>
+            </modules>
+        </profile>
         <profile>
             <id>all</id>
             <modules>
@@ -114,6 +124,7 @@
                 <module>samoa-flink</module>
                 <module>samoa-samza</module>
                 <module>samoa-test</module>
+                <module>samoa-heron</module>
             </modules>
         </profile>
     </profiles>
@@ -144,6 +155,8 @@
         <storm.version>0.9.4</storm.version>
         <!-- storm 0.8.2 loads zookeeper classes with hardcoded names from 3.3 version-->
         <zookeeper.storm.version>3.4.6</zookeeper.storm.version>
+        <heron.version>0.17.8</heron.version>
+        <zookeeper.heron.version>3.4.6</zookeeper.heron.version>
         <avro.version>1.7.7</avro.version>
     </properties>
 
@@ -198,7 +211,8 @@
                     <nohelp>true</nohelp>
                     <header>Apache SAMOA ${project.version}</header>
                     <footer>Scalable Advanced Massive Online Analysis,
-                        ${project.version}</footer>
+                        ${project.version}
+                    </footer>
                     <doctitle>Apache SAMOA API ${project.version}</doctitle>
                     <links>
                         <link>http://samoa.incubator.apache.org/docs/api/</link>
@@ -270,7 +284,7 @@
                                         </goals>
                                     </pluginExecutionFilter>
                                     <action>
-                                        <ignore />
+                                        <ignore/>
                                     </action>
                                 </pluginExecution>
                             </pluginExecutions>

samoa-api/src/main/java/org/apache/samoa/moa/core/Vote.java

@@ -1,5 +1,25 @@
 package org.apache.samoa.moa.core;
 
+/*
+ * #%L
+ * SAMOA
+ * %%
+ * Copyright (C) 2014 - 2018 Apache Software Foundation
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
 import java.io.Serializable;
 
 /*

samoa-heron/pom.xml

@@ -0,0 +1,151 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  #%L
+  SAMOA
+  %%
+  Copyright (C) 2018 Apache Software Foundation
+  %%
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  #L%
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <properties>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <name>samoa-heron</name>
+    <description>Heron bindings for SAMOA</description>
+
+    <artifactId>samoa-heron</artifactId>
+    <parent>
+        <groupId>org.apache.samoa</groupId>
+        <artifactId>samoa</artifactId>
+        <version>0.5.0-incubating-SNAPSHOT</version>
+    </parent>
+
+
+  <repositories>
+    <repository>
+      <id>mvnrepository</id>
+      <name>Mvn repository</name>
+      <url>https://mvnrepository.com/artifact/</url>
+      <releases>
+        <enabled>false</enabled>
+      </releases>
+      <snapshots>
+        <enabled>true</enabled>
+      </snapshots>
+    </repository>
+  </repositories>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.samoa</groupId>
+      <artifactId>samoa-api</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.samoa</groupId>
+      <artifactId>samoa-test</artifactId>
+      <type>test-jar</type>
+      <classifier>test-jar-with-dependencies</classifier>
+      <version>${project.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.twitter.heron</groupId>
+      <artifactId>heron-storm</artifactId>
+      <version>${heron.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.zookeeper</groupId>
+      <artifactId>zookeeper</artifactId>
+      <version>${zookeeper.heron.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-log4j12</artifactId>
+      <version>${slf4j-log4j12.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.googlecode.json-simple</groupId>
+      <artifactId>json-simple</artifactId>
+      <version>1.1</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.thrift</groupId>
+      <artifactId>libthrift</artifactId>
+      <version>0.11.0</version>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <!-- SAMOA assembly -->
+      <plugin>
+        <artifactId>maven-assembly-plugin</artifactId>
+        <version>${maven-assembly-plugin.version}</version>
+        <configuration>
+          <finalName>SAMOA-heron-${project.version}</finalName>
+          <appendAssemblyId>false</appendAssemblyId>
+          <attach>false</attach>
+          <outputDirectory>../target</outputDirectory>
+          <descriptorRefs>
+            <descriptorRef>jar-with-dependencies</descriptorRef>
+          </descriptorRefs>
+          <archive>
+            <manifestEntries>
+              <Bundle-Version>${parsedVersion.osgiVersion}</Bundle-Version>
+              <Bundle-Description>${project.description}</Bundle-Description>
+              <Implementation-Version>${project.version}</Implementation-Version>
+              <Implementation-Vendor>Yahoo Labs</Implementation-Vendor>
+              <Implementation-Vendor-Id>SAMOA</Implementation-Vendor-Id>
+            </manifestEntries>
+          </archive>
+        </configuration>
+        <executions>
+          <execution>
+            <id>make-assembly</id> <!-- this is used for inheritance merges -->
+            <phase>package</phase>
+            <goals>
+              <goal>single</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-surefire-plugin</artifactId>
+        <version>${maven-surefire-plugin.version}</version>
+        <configuration>
+          <argLine>-Xmx1G</argLine>
+          <redirectTestOutputToFile>false</redirectTestOutputToFile>
+        </configuration>
+      </plugin>
+    </plugins>
+   <testResources>
+    <testResource>
+      <directory>${project.basedir}/../bin</directory>
+      <includes>
+        <include>*heron.properties</include>
+      </includes>
+    </testResource>
+   </testResources>
+  </build>
+</project>

samoa-heron/src/main/java/org/apache/samoa/LocalHeronDoTask.java

@@ -0,0 +1,74 @@
+package org.apache.samoa.heron.topology;
+/*
+ * #%L
+ * SAMOA
+ * %%
+ * Copyright (C) 2014 - 2015 Apache Software Foundation
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.samoa.heron.topology.impl.HeronSamoaUtils;
+import org.apache.samoa.heron.topology.impl.HeronTopology;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.commons.configuration.Configuration;
+import com.twitter.heron.api.Config;
+import com.twitter.heron.api.utils.Utils;
+import com.twitter.heron.api.exception.AlreadyAliveException;
+import com.twitter.heron.api.exception.InvalidTopologyException;
+import com.twitter.heron.simulator.Simulator;
+
+
+/**
+ * The main class to execute a SAMOA task in LOCAL mode in Heron.
+ *
+ * @author Arinto Murdopo
+ */
+public class LocalHeronDoTask {
+    private static final Logger logger = LoggerFactory.getLogger(LocalHeronDoTask.class);
+    private static final String EXECUTION_DURATION_KEY = "samoa.storm.local.mode.execution.duration";
+    private static final String SAMOA_STORM_PROPERTY_FILE_LOC = "samoa-heron.properties";
+
+    /**
+     * The main method.
+     *
+     * @param args the arguments
+     */
+    public static void main(String[] args) {
+        List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args));
+        int numWorker = HeronSamoaUtils.numWorkers(tmpArgs);
+        args = tmpArgs.toArray(new String[0]);
+        // convert the arguments into Storm topology
+        HeronTopology heronTopo = HeronSamoaUtils.argsToTopology(args);
+        String topologyName = heronTopo.getTopologyName();
+        Config conf = new Config();
+        // conf.putAll(Utils.readStormConfig());
+        conf.setDebug(false);
+        // local mode
+        //conf.setMaxTaskParallelism(numWorker);
+        Simulator cluster = new Simulator();
+        cluster.submitTopology(topologyName, conf, heronTopo.getHeronBuilder().createTopology());
+        // Read local mode execution duration from property file
+        Configuration heronConfig = HeronSamoaUtils.getPropertyConfig(LocalHeronDoTask.SAMOA_STORM_PROPERTY_FILE_LOC);
+        long executionDuration = heronConfig.getLong(LocalHeronDoTask.EXECUTION_DURATION_KEY);
+        backtype.storm.utils.Utils.sleep(executionDuration * 1000);
+        cluster.killTopology(topologyName);
+        cluster.shutdown();
+    }
+}