Skip to content

Girish A Nandolkar / grpc-project

Go to a project
Commit 988e94c9 by Girish A Nandolkar
Options

synchronus job creation and getting the result back is done and asynchronus response is done

1 parent 3d6f5032
Inline Side-by-side
Showing with 393 additions and 213 deletions
.idea/modules/newscalaexp.iml
......@@ -24,83 +24,83 @@
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="SBT: com.google.api.grpc:proto-google-common-protos:0.1.9:jar" level="project" />
<orderEntry type="library" name="SBT: org.json4s:json4s-scalap_2.12:3.5.3:jar" level="project" />
<orderEntry type="library" name="SBT: org.json4s:json4s-native_2.12:3.5.3:jar" level="project" />
<orderEntry type="library" name="SBT: org.json4s:json4s-core_2.12:3.5.3:jar" level="project" />
<orderEntry type="library" name="SBT: org.json4s:json4s-ast_2.12:3.5.3:jar" level="project" />
<orderEntry type="library" name="SBT: com.thoughtworks.paranamer:paranamer:2.8:jar" level="project" />
<orderEntry type="library" name="SBT: com.softwaremill.sttp:json4s_2.12:1.1.0:jar" level="project" />
<orderEntry type="library" name="SBT: com.softwaremill.sttp:core_2.12:1.1.0:jar" level="project" />
<orderEntry type="library" name="SBT: org.scala-lang:scala-library:2.12.4:jar" level="project" />
<orderEntry type="library" name="SBT: io.opencensus:opencensus-contrib-grpc-metrics:0.8.0:jar" level="project" />
<orderEntry type="library" name="SBT: io.opencensus:opencensus-api:0.8.0:jar" level="project" />
<orderEntry type="library" name="SBT: io.netty:netty-transport:4.1.16.Final:jar" level="project" />
<orderEntry type="library" name="SBT: io.netty:netty-resolver:4.1.16.Final:jar" level="project" />
<orderEntry type="library" name="SBT: io.netty:netty-handler-proxy:4.1.16.Final:jar" level="project" />
<orderEntry type="library" name="SBT: io.netty:netty-handler:4.1.16.Final:jar" level="project" />
<orderEntry type="library" name="SBT: io.netty:netty-common:4.1.16.Final:jar" level="project" />
<orderEntry type="library" name="SBT: io.netty:netty-codec-socks:4.1.16.Final:jar" level="project" />
<orderEntry type="library" name="SBT: io.netty:netty-codec-http2:4.1.16.Final:jar" level="project" />
<orderEntry type="library" name="SBT: io.netty:netty-codec-http:4.1.16.Final:jar" level="project" />
<orderEntry type="library" name="SBT: io.netty:netty-codec:4.1.16.Final:jar" level="project" />
<orderEntry type="library" name="SBT: io.netty:netty-buffer:4.1.16.Final:jar" level="project" />
<orderEntry type="library" name="SBT: io.grpc:grpc-stub:1.8.0:jar" level="project" />
<orderEntry type="library" name="SBT: io.grpc:grpc-protobuf-lite:1.8.0:jar" level="project" />
<orderEntry type="library" name="SBT: io.grpc:grpc-protobuf:1.8.0:jar" level="project" />
<orderEntry type="library" name="SBT: io.grpc:grpc-netty:1.8.0:jar" level="project" />
<orderEntry type="library" name="SBT: io.grpc:grpc-core:1.8.0:jar" level="project" />
<orderEntry type="library" name="SBT: io.grpc:grpc-context:1.8.0:jar" level="project" />
<orderEntry type="library" name="SBT: com.trueaccord.scalapb:scalapb-runtime_2.12:0.6.7:jar" level="project" />
<orderEntry type="library" name="SBT: com.trueaccord.scalapb:scalapb-runtime-grpc_2.12:0.6.7:jar" level="project" />
<orderEntry type="library" name="SBT: com.trueaccord.lenses:lenses_2.12:0.4.12:jar" level="project" />
<orderEntry type="library" name="SBT: com.lihaoyi:sourcecode_2.12:0.1.4:jar" level="project" />
<orderEntry type="library" name="SBT: com.lihaoyi:fastparse_2.12:1.0.0:jar" level="project" />
<orderEntry type="library" name="SBT: com.lihaoyi:fastparse-utils_2.12:1.0.0:jar" level="project" />
<orderEntry type="library" name="SBT: com.google.protobuf:protobuf-java-util:3.4.0:jar" level="project" />
<orderEntry type="library" name="SBT: com.google.protobuf:protobuf-java:3.4.0:jar" level="project" />
<orderEntry type="library" name="SBT: com.google.instrumentation:instrumentation-api:0.4.3:jar" level="project" />
<orderEntry type="library" name="SBT: com.google.errorprone:error_prone_annotations:2.0.19:jar" level="project" />
<orderEntry type="library" name="SBT: com.google.code.gson:gson:2.7:jar" level="project" />
<orderEntry type="library" name="SBT: com.google.code.findbugs:jsr305:3.0.0:jar" level="project" />
<orderEntry type="library" name="SBT: com.fasterxml.jackson.core:jackson-annotations:2.8.10:jar" level="project" />
<orderEntry type="library" name="SBT: com.fasterxml.jackson.core:jackson-core:2.8.10:jar" level="project" />
<orderEntry type="library" name="SBT: com.fasterxml.jackson.core:jackson-databind:2.8.10:jar" level="project" />
<orderEntry type="library" name="SBT: com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.8.10:jar" level="project" />
<orderEntry type="library" name="SBT: com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.8.10:jar" level="project" />
<orderEntry type="library" name="SBT: com.google.guava:guava:22.0:jar" level="project" />
<orderEntry type="library" name="SBT: com.google.j2objc:j2objc-annotations:1.1:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe:config:1.3.1:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe:ssl-config-core_2.12:0.2.2:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.akka:akka-actor_2.12:2.5.6:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.akka:akka-slf4j_2.12:2.5.6:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.akka:akka-stream_2.12:2.5.6:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:build-link:2.6.7:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:play-exceptions:2.6.7:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:play-functional_2.12:2.6.7:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:play-json_2.12:2.6.7:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:play-netty-utils:2.6.7:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:play-streams_2.12:2.6.7:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:play-ws-standalone-json_2.12:1.1.3:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:play-ws-standalone-xml_2.12:1.1.3:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:play-ws-standalone_2.12:1.1.3:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:play-ws_2.12:2.6.7:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:play_2.12:2.6.7:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:twirl-api_2.12:1.3.12:jar" level="project" />
<orderEntry type="library" name="SBT: commons-codec:commons-codec:1.10:jar" level="project" />
<orderEntry type="library" name="SBT: io.jsonwebtoken:jjwt:0.7.0:jar" level="project" />
<orderEntry type="library" name="SBT: javax.inject:javax.inject:1:jar" level="project" />
<orderEntry type="library" name="SBT: javax.transaction:jta:1.1:jar" level="project" />
<orderEntry type="library" name="SBT: joda-time:joda-time:2.9.9:jar" level="project" />
<orderEntry type="library" name="SBT: org.apache.commons:commons-lang3:3.6:jar" level="project" />
<orderEntry type="library" name="SBT: org.codehaus.mojo:animal-sniffer-annotations:1.14:jar" level="project" />
<orderEntry type="library" name="SBT: org.reactivestreams:reactive-streams:1.0.1:jar" level="project" />
<orderEntry type="library" name="SBT: org.scala-lang:scala-reflect:2.12.4:jar" level="project" />
<orderEntry type="library" name="SBT: org.scala-lang.modules:scala-java8-compat_2.12:0.8.0:jar" level="project" />
<orderEntry type="library" name="SBT: org.scala-lang.modules:scala-parser-combinators_2.12:1.0.6:jar" level="project" />
<orderEntry type="library" name="SBT: org.scala-lang.modules:scala-xml_2.12:1.0.6:jar" level="project" />
<orderEntry type="library" name="SBT: org.slf4j:jcl-over-slf4j:1.7.25:jar" level="project" />
<orderEntry type="library" name="SBT: org.slf4j:jul-to-slf4j:1.7.25:jar" level="project" />
<orderEntry type="library" name="SBT: org.slf4j:slf4j-api:1.7.25:jar" level="project" />
<orderEntry type="library" name="SBT: org.typelevel:macro-compat_2.12:1.1.1:jar" level="project" />
<orderEntry type="library" name="SBT: org.slf4j:slf4j-api:1.7.25:jar" level="project" />
<orderEntry type="library" name="SBT: org.slf4j:jul-to-slf4j:1.7.25:jar" level="project" />
<orderEntry type="library" name="SBT: org.slf4j:jcl-over-slf4j:1.7.25:jar" level="project" />
<orderEntry type="library" name="SBT: org.scala-lang.modules:scala-xml_2.12:1.0.6:jar" level="project" />
<orderEntry type="library" name="SBT: org.scala-lang.modules:scala-parser-combinators_2.12:1.0.6:jar" level="project" />
<orderEntry type="library" name="SBT: org.scala-lang.modules:scala-java8-compat_2.12:0.8.0:jar" level="project" />
<orderEntry type="library" name="SBT: org.scala-lang:scala-reflect:2.12.4:jar" level="project" />
<orderEntry type="library" name="SBT: org.reactivestreams:reactive-streams:1.0.1:jar" level="project" />
<orderEntry type="library" name="SBT: org.codehaus.mojo:animal-sniffer-annotations:1.14:jar" level="project" />
<orderEntry type="library" name="SBT: org.apache.commons:commons-lang3:3.6:jar" level="project" />
<orderEntry type="library" name="SBT: joda-time:joda-time:2.9.9:jar" level="project" />
<orderEntry type="library" name="SBT: javax.transaction:jta:1.1:jar" level="project" />
<orderEntry type="library" name="SBT: javax.inject:javax.inject:1:jar" level="project" />
<orderEntry type="library" name="SBT: io.jsonwebtoken:jjwt:0.7.0:jar" level="project" />
<orderEntry type="library" name="SBT: commons-codec:commons-codec:1.10:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:twirl-api_2.12:1.3.12:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:play_2.12:2.6.7:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:play-ws_2.12:2.6.7:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:play-ws-standalone_2.12:1.1.3:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:play-ws-standalone-xml_2.12:1.1.3:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:play-ws-standalone-json_2.12:1.1.3:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:play-streams_2.12:2.6.7:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:play-netty-utils:2.6.7:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:play-json_2.12:2.6.7:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:play-functional_2.12:2.6.7:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:play-exceptions:2.6.7:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.play:build-link:2.6.7:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.akka:akka-stream_2.12:2.5.6:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.akka:akka-slf4j_2.12:2.5.6:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe.akka:akka-actor_2.12:2.5.6:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe:ssl-config-core_2.12:0.2.2:jar" level="project" />
<orderEntry type="library" name="SBT: com.typesafe:config:1.3.1:jar" level="project" />
<orderEntry type="library" name="SBT: com.google.j2objc:j2objc-annotations:1.1:jar" level="project" />
<orderEntry type="library" name="SBT: com.google.guava:guava:22.0:jar" level="project" />
<orderEntry type="library" name="SBT: com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.8.10:jar" level="project" />
<orderEntry type="library" name="SBT: com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.8.10:jar" level="project" />
<orderEntry type="library" name="SBT: com.fasterxml.jackson.core:jackson-databind:2.8.10:jar" level="project" />
<orderEntry type="library" name="SBT: com.fasterxml.jackson.core:jackson-core:2.8.10:jar" level="project" />
<orderEntry type="library" name="SBT: com.fasterxml.jackson.core:jackson-annotations:2.8.10:jar" level="project" />
<orderEntry type="library" name="SBT: com.google.code.findbugs:jsr305:3.0.0:jar" level="project" />
<orderEntry type="library" name="SBT: com.google.code.gson:gson:2.7:jar" level="project" />
<orderEntry type="library" name="SBT: com.google.errorprone:error_prone_annotations:2.0.19:jar" level="project" />
<orderEntry type="library" name="SBT: com.google.instrumentation:instrumentation-api:0.4.3:jar" level="project" />
<orderEntry type="library" name="SBT: com.google.protobuf:protobuf-java:3.4.0:jar" level="project" />
<orderEntry type="library" name="SBT: com.google.protobuf:protobuf-java-util:3.4.0:jar" level="project" />
<orderEntry type="library" name="SBT: com.lihaoyi:fastparse-utils_2.12:1.0.0:jar" level="project" />
<orderEntry type="library" name="SBT: com.lihaoyi:fastparse_2.12:1.0.0:jar" level="project" />
<orderEntry type="library" name="SBT: com.lihaoyi:sourcecode_2.12:0.1.4:jar" level="project" />
<orderEntry type="library" name="SBT: com.trueaccord.lenses:lenses_2.12:0.4.12:jar" level="project" />
<orderEntry type="library" name="SBT: com.trueaccord.scalapb:scalapb-runtime-grpc_2.12:0.6.7:jar" level="project" />
<orderEntry type="library" name="SBT: com.trueaccord.scalapb:scalapb-runtime_2.12:0.6.7:jar" level="project" />
<orderEntry type="library" name="SBT: io.grpc:grpc-context:1.8.0:jar" level="project" />
<orderEntry type="library" name="SBT: io.grpc:grpc-core:1.8.0:jar" level="project" />
<orderEntry type="library" name="SBT: io.grpc:grpc-netty:1.8.0:jar" level="project" />
<orderEntry type="library" name="SBT: io.grpc:grpc-protobuf:1.8.0:jar" level="project" />
<orderEntry type="library" name="SBT: io.grpc:grpc-protobuf-lite:1.8.0:jar" level="project" />
<orderEntry type="library" name="SBT: io.grpc:grpc-stub:1.8.0:jar" level="project" />
<orderEntry type="library" name="SBT: io.netty:netty-buffer:4.1.16.Final:jar" level="project" />
<orderEntry type="library" name="SBT: io.netty:netty-codec:4.1.16.Final:jar" level="project" />
<orderEntry type="library" name="SBT: io.netty:netty-codec-http:4.1.16.Final:jar" level="project" />
<orderEntry type="library" name="SBT: io.netty:netty-codec-http2:4.1.16.Final:jar" level="project" />
<orderEntry type="library" name="SBT: io.netty:netty-codec-socks:4.1.16.Final:jar" level="project" />
<orderEntry type="library" name="SBT: io.netty:netty-common:4.1.16.Final:jar" level="project" />
<orderEntry type="library" name="SBT: io.netty:netty-handler:4.1.16.Final:jar" level="project" />
<orderEntry type="library" name="SBT: io.netty:netty-handler-proxy:4.1.16.Final:jar" level="project" />
<orderEntry type="library" name="SBT: io.netty:netty-resolver:4.1.16.Final:jar" level="project" />
<orderEntry type="library" name="SBT: io.netty:netty-transport:4.1.16.Final:jar" level="project" />
<orderEntry type="library" name="SBT: io.opencensus:opencensus-api:0.8.0:jar" level="project" />
<orderEntry type="library" name="SBT: io.opencensus:opencensus-contrib-grpc-metrics:0.8.0:jar" level="project" />
<orderEntry type="library" name="SBT: org.scala-lang:scala-library:2.12.4:jar" level="project" />
<orderEntry type="library" name="SBT: com.softwaremill.sttp:core_2.12:1.1.0:jar" level="project" />
<orderEntry type="library" name="SBT: com.softwaremill.sttp:json4s_2.12:1.1.0:jar" level="project" />
<orderEntry type="library" name="SBT: com.thoughtworks.paranamer:paranamer:2.8:jar" level="project" />
<orderEntry type="library" name="SBT: org.json4s:json4s-ast_2.12:3.5.3:jar" level="project" />
<orderEntry type="library" name="SBT: org.json4s:json4s-core_2.12:3.5.3:jar" level="project" />
<orderEntry type="library" name="SBT: org.json4s:json4s-native_2.12:3.5.3:jar" level="project" />
<orderEntry type="library" name="SBT: org.json4s:json4s-scalap_2.12:3.5.3:jar" level="project" />
</component>
</module>
\ No newline at end of file
.idea/workspace.xml
......@@ -2,6 +2,9 @@
<project version="4">
<component name="ChangeListManager">
<list default="true" id="7b04a7b7-eb3d-45db-8e9b-8c734276f9c0" name="Default" comment="">
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/.idea/modules/newscalaexp.iml" afterPath="$PROJECT_DIR$/.idea/modules/newscalaexp.iml" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/build.sbt" afterPath="$PROJECT_DIR$/build.sbt" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/src/main/protobuf/sparkJob.proto" afterPath="$PROJECT_DIR$/src/main/protobuf/sparkJob.proto" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/src/main/scala/CreateSparkJobClient.scala" afterPath="$PROJECT_DIR$/src/main/scala/CreateSparkJobClient.scala" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/src/main/scala/CreateSparkJobServer.scala" afterPath="$PROJECT_DIR$/src/main/scala/CreateSparkJobServer.scala" />
......@@ -43,8 +46,8 @@
<file leaf-file-name="sparkJob.proto" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/protobuf/sparkJob.proto">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="713">
<caret line="31" column="1" lean-forward="true" selection-start-line="31" selection-start-column="1" selection-end-line="31" selection-end-column="1" />
<state relative-caret-position="276">
<caret line="12" column="23" lean-forward="false" selection-start-line="12" selection-start-column="8" selection-end-line="12" selection-end-column="23" />
<folding />
</state>
</provider>
......@@ -53,8 +56,8 @@
<file leaf-file-name="CreateSparkJobClient.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/CreateSparkJobClient.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="69">
<caret line="3" column="0" lean-forward="true" selection-start-line="3" selection-start-column="0" selection-end-line="3" selection-end-column="0" />
<state relative-caret-position="92">
<caret line="4" column="0" lean-forward="true" selection-start-line="4" selection-start-column="0" selection-end-line="4" selection-end-column="0" />
<folding>
<element signature="e#0#98#0" expanded="true" />
</folding>
......@@ -65,22 +68,10 @@
<file leaf-file-name="CreateSparkJobServer.scala" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/main/scala/CreateSparkJobServer.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="509">
<caret line="58" column="21" lean-forward="true" selection-start-line="58" selection-start-column="21" selection-end-line="58" selection-end-column="21" />
<state relative-caret-position="-2909">
<caret line="23" column="5" lean-forward="false" selection-start-line="23" selection-start-column="5" selection-end-line="23" selection-end-column="5" />
<folding>
<element signature="e#2#28#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file leaf-file-name="Models.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/Models.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="633">
<caret line="32" column="23" lean-forward="false" selection-start-line="32" selection-start-column="23" selection-end-line="32" selection-end-column="23" />
<folding>
<element signature="e#761#796#0" expanded="true" />
<element signature="e#3#39#0" expanded="true" />
</folding>
</state>
</provider>
......@@ -89,8 +80,8 @@
<file leaf-file-name="build.sbt" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/build.sbt">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="322">
<caret line="14" column="48" lean-forward="false" selection-start-line="14" selection-start-column="48" selection-end-line="14" selection-end-column="48" />
<state relative-caret-position="161">
<caret line="7" column="2" lean-forward="false" selection-start-line="7" selection-start-column="2" selection-end-line="7" selection-end-column="2" />
<folding />
</state>
</provider>
......@@ -106,6 +97,18 @@
</provider>
</entry>
</file>
<file leaf-file-name="Models.scala" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/src/main/scala/Models.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="506">
<caret line="22" column="58" lean-forward="false" selection-start-line="22" selection-start-column="58" selection-end-line="22" selection-end-column="58" />
<folding>
<element signature="e#761#796#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="Git.Settings">
......@@ -120,8 +123,8 @@
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/src/main/scala/TestClient.scala" />
<option value="$PROJECT_DIR$/build.sbt" />
<option value="$PROJECT_DIR$/src/main/scala/Models.scala" />
<option value="$PROJECT_DIR$/build.sbt" />
<option value="$PROJECT_DIR$/src/main/protobuf/sparkJob.proto" />
<option value="$PROJECT_DIR$/src/main/scala/CreateSparkJobClient.scala" />
<option value="$PROJECT_DIR$/src/main/scala/CreateSparkJobServer.scala" />
......@@ -136,7 +139,7 @@
</component>
<component name="ProjectLevelVcsManager" settingsEditedManually="true" />
<component name="ProjectView">
<navigator currentView="ProjectPane" proportions="" version="1">
<navigator proportions="" version="1">
<flattenPackages />
<showMembers />
<showModules />
......@@ -149,53 +152,7 @@
<manualOrder />
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="Scratches" />
<pane id="ProjectPane">
<subPane>
<expand>
<path>
<item name="newScalaExp" type="b2602c69:ProjectViewProjectNode" />
<item name="newScalaExp" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="newScalaExp" type="b2602c69:ProjectViewProjectNode" />
<item name="newScalaExp" type="462c0819:PsiDirectoryNode" />
<item name="project" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="newScalaExp" type="b2602c69:ProjectViewProjectNode" />
<item name="newScalaExp" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="newScalaExp" type="b2602c69:ProjectViewProjectNode" />
<item name="newScalaExp" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="newScalaExp" type="b2602c69:ProjectViewProjectNode" />
<item name="newScalaExp" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="protobuf" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="newScalaExp" type="b2602c69:ProjectViewProjectNode" />
<item name="newScalaExp" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="scala" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
</pane>
<pane id="PackagesPane" />
<pane id="AndroidView" />
<pane id="Scope" />
</panes>
<panes />
</component>
<component name="PropertiesComponent">
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
......@@ -248,6 +205,7 @@
<env name="RAPIDOR_CURRENT_DB_NAME" value="rapidor_realmarketing" />
<env name="RAPIDOR_PLANX_DB_PASSWORD" value="%5Bguessme321%2Bguessme321%5D" />
<env name="SPARK_APP_RESOURCE" value="file:/home/acelr/spark/jar/rapidor-spark_2.11-1.0.jar" />
<env name="SHEDULE_TIME_FOR_STATUS" value="1200000" />
</envs>
</configuration>
<configuration name="TestClient" type="Application" factoryName="Application" temporary="true">
......@@ -375,7 +333,7 @@
</recent_temporary>
</component>
<component name="SbtLocalSettings">
<option name="lastUpdateTimestamp" value="1511939772006" />
<option name="lastUpdateTimestamp" value="1512030506657" />
<option name="availableProjects">
<map>
<entry>
......@@ -423,7 +381,7 @@
<layout>
<window_info id="Palette" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
<window_info id="Messages" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32984293" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Messages" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32984293" sideWeight="0.5" order="9" side_tool="false" content_ui="tabs" />
<window_info id="SBT" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32946378" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Palette&#9;" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Image Layers" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
......@@ -431,12 +389,12 @@
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" />
<window_info id="Maven Projects" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.634555" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="8" side_tool="false" content_ui="tabs" />
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="10" side_tool="false" content_ui="tabs" />
<window_info id="Capture Tool" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.1840796" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="sbt-shell-toolwindow" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="-1" side_tool="false" content_ui="tabs" />
<window_info id="sbt-shell-toolwindow" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="11" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="UI Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
......@@ -496,6 +454,60 @@
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/src/main/protobuf/sparkJob.proto">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="713">
<caret line="31" column="1" lean-forward="true" selection-start-line="31" selection-start-column="1" selection-end-line="31" selection-end-column="1" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/CreateSparkJobClient.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="23">
<caret line="3" column="0" lean-forward="true" selection-start-line="3" selection-start-column="0" selection-end-line="3" selection-end-column="0" />
<folding>
<element signature="e#0#98#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/Models.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="736">
<caret line="32" column="23" lean-forward="false" selection-start-line="32" selection-start-column="23" selection-end-line="32" selection-end-column="23" />
<folding>
<element signature="e#761#796#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/build.sbt">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="322">
<caret line="14" column="48" lean-forward="false" selection-start-line="14" selection-start-column="48" selection-end-line="14" selection-end-column="48" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/project/plugins.sbt">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="0">
<caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/CreateSparkJobServer.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="1334">
<caret line="58" column="21" lean-forward="true" selection-start-line="58" selection-start-column="21" selection-end-line="58" selection-end-column="21" />
<folding>
<element signature="e#3#39#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/protobuf/sparkJob.proto">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="552">
<caret line="24" column="16" lean-forward="false" selection-start-line="24" selection-start-column="9" selection-end-line="24" selection-end-column="16" />
<folding />
......@@ -541,7 +553,7 @@
<state relative-caret-position="46">
<caret line="5" column="44" lean-forward="false" selection-start-line="5" selection-start-column="44" selection-end-line="5" selection-end-column="44" />
<folding>
<element signature="e#0#51#0" expanded="true" />
<element signature="e#0#51#0" expanded="false" />
</folding>
</state>
</provider>
......@@ -550,7 +562,6 @@
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="69">
<caret line="3" column="0" lean-forward="false" selection-start-line="3" selection-start-column="0" selection-end-line="3" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
......@@ -559,7 +570,7 @@
<state relative-caret-position="1242">
<caret line="63" column="69" lean-forward="false" selection-start-line="63" selection-start-column="69" selection-end-line="63" selection-end-column="69" />
<folding>
<element signature="e#2#28#0" expanded="true" />
<element signature="e#3#39#0" expanded="true" />
</folding>
</state>
</provider>
......@@ -568,7 +579,6 @@
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="69">
<caret line="3" column="0" lean-forward="false" selection-start-line="3" selection-start-column="0" selection-end-line="3" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
......@@ -577,7 +587,7 @@
<state relative-caret-position="46">
<caret line="5" column="44" lean-forward="false" selection-start-line="5" selection-start-column="44" selection-end-line="5" selection-end-column="44" />
<folding>
<element signature="e#0#51#0" expanded="true" />
<element signature="e#0#51#0" expanded="false" />
</folding>
</state>
</provider>
......@@ -590,22 +600,7 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/build.sbt">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="322">
<caret line="14" column="48" lean-forward="false" selection-start-line="14" selection-start-column="48" selection-end-line="14" selection-end-column="48" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/Models/sca.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="46">
<caret line="2" column="6" lean-forward="false" selection-start-line="2" selection-start-column="6" selection-end-line="2" selection-end-column="6" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/Models/sca.scala" />
<entry file="file://$PROJECT_DIR$/src/main/scala/TestServerConnect.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="230">
......@@ -618,7 +613,6 @@
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="267">
<caret line="38" column="0" lean-forward="false" selection-start-line="38" selection-start-column="0" selection-end-line="38" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
......@@ -626,7 +620,6 @@
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="135">
<caret line="172" column="0" lean-forward="false" selection-start-line="172" selection-start-column="0" selection-end-line="172" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
......@@ -646,40 +639,64 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/Models.scala">
<entry file="jar://$USER_HOME$/.ivy2/cache/com.softwaremill.sttp/core_2.12/srcs/core_2.12-1.1.0-sources.jar!/com/softwaremill/sttp/ResponseAs.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="633">
<caret line="32" column="23" lean-forward="false" selection-start-line="32" selection-start-column="23" selection-end-line="32" selection-end-column="23" />
<folding>
<element signature="e#761#796#0" expanded="true" />
</folding>
<state relative-caret-position="300">
<caret line="21" column="13" lean-forward="false" selection-start-line="21" selection-start-column="13" selection-end-line="21" selection-end-column="13" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/CreateSparkJobClient.scala">
<entry file="jar://$USER_HOME$/.ivy2/cache/org.scala-lang/scala-library/srcs/scala-library-2.12.4-sources.jar!/scala/collection/TraversableLike.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="69">
<caret line="3" column="0" lean-forward="true" selection-start-line="3" selection-start-column="0" selection-end-line="3" selection-end-column="0" />
<state relative-caret-position="106">
<caret line="142" column="6" lean-forward="false" selection-start-line="142" selection-start-column="6" selection-end-line="142" selection-end-column="6" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/build.sbt">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="161">
<caret line="7" column="2" lean-forward="false" selection-start-line="7" selection-start-column="2" selection-end-line="7" selection-end-column="2" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/Models.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="506">
<caret line="22" column="58" lean-forward="false" selection-start-line="22" selection-start-column="58" selection-end-line="22" selection-end-column="58" />
<folding>
<element signature="e#0#98#0" expanded="true" />
<element signature="e#761#796#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/protobuf/sparkJob.proto">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="713">
<caret line="31" column="1" lean-forward="true" selection-start-line="31" selection-start-column="1" selection-end-line="31" selection-end-column="1" />
<state relative-caret-position="276">
<caret line="12" column="23" lean-forward="false" selection-start-line="12" selection-start-column="8" selection-end-line="12" selection-end-column="23" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/CreateSparkJobClient.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="92">
<caret line="4" column="0" lean-forward="true" selection-start-line="4" selection-start-column="0" selection-end-line="4" selection-end-column="0" />
<folding>
<element signature="e#0#98#0" expanded="true" />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/CreateSparkJobServer.scala">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="509">
<caret line="58" column="21" lean-forward="true" selection-start-line="58" selection-start-column="21" selection-end-line="58" selection-end-column="21" />
<state relative-caret-position="-2909">
<caret line="23" column="5" lean-forward="false" selection-start-line="23" selection-start-column="5" selection-end-line="23" selection-end-column="5" />
<folding>
<element signature="e#2#28#0" expanded="true" />
<element signature="e#3#39#0" expanded="true" />
</folding>
</state>
</provider>
......
build.sbt
......@@ -5,6 +5,9 @@ version := "0.1"
scalaVersion := "2.12.4"
//resolvers += "theatr.us" at "http://repo.theatr.us"
PB.targets in Compile := Seq(
scalapb.gen() -> (sourceManaged in Compile).value
)
......@@ -13,6 +16,8 @@ libraryDependencies ++= Seq(
"com.typesafe.play" %% "play-ws" % "2.6.7",
"com.softwaremill.sttp" %% "core" % "1.1.0",
"com.softwaremill.sttp" %% "json4s" % "1.1.0",
"com.typesafe.akka" %% "akka-actor" % "2.4.8",
// "us.theatr" %% "akka-quartz" % "0.3.0",
"io.grpc" % "grpc-netty" % com.trueaccord.scalapb.compiler.Version.grpcJavaVersion,
"com.trueaccord.scalapb" %% "scalapb-runtime" % com.trueaccord.scalapb.compiler.Version.scalapbVersion % "protobuf",
"com.trueaccord.scalapb" %% "scalapb-runtime-grpc" % com.trueaccord.scalapb.compiler.Version.scalapbVersion
......
src/main/protobuf/sparkJob.proto
......@@ -4,7 +4,9 @@ package com.example.protos;
service CreateSparkJob {
rpc CreateJob (SparkJobRequest) returns (SparkJobReply) {}
rpc SparkJobStatus (SparkJobStatusReq) returns (SparkJobReply) {}
rpc SparkJobStatus (SparkJobStatusReq) returns (SparkJobStatusReply) {}
rpc SparkJobResponse (SparkJobResponseReq) returns (SparkJobResponseReply) {}
rpc CreateJobAndGetResult(SparkJobRequest) returns (SparkJobResponseReply) {}
}
// The request message containing the debug, dbName, filename, reporttype, instanceDbs, startDate,endDate, reportGenerateType,tallyCompanyName,allowCatogery.
......@@ -18,15 +20,28 @@ message SparkJobRequest {
string endDate = 7;
string reportGenerateType = 8;
string tallyCompanyName = 9;
string allowedCatogery = 10;
}
// The response message containing the greetings
message SparkJobReply {
string message = 1;
string submissionId = 1;
}
message SparkJobStatusReq{
string submitionId = 1;
string submissionId = 1;
}
message SparkJobStatusReply{
string status = 1;
}
message SparkJobResponseReq{
string submissionId = 1;
string filePath = 2;
}
message SparkJobResponseReply{
string status = 1;
string message = 2;
}
src/main/scala/CreateSparkJobClient.scala
import com.example.protos.sparkJob.CreateSparkJobGrpc.{CreateSparkJob, CreateSparkJobBlockingStub}
import com.example.protos.sparkJob.{CreateSparkJobGrpc, SparkJobReply, SparkJobRequest, SparkJobStatusReq}
import com.example.protos.sparkJob._
import io.grpc.ManagedChannelBuilder
......@@ -9,17 +9,25 @@ object CreateSparkJobClient {
val channel = ManagedChannelBuilder.forAddress("localhost",50051)
.usePlaintext(true).build()
val request = SparkJobRequest("true","rapidor_realmarketing","abc.csv","payment_collection","none","01/02/2017","15/02/2017","","","")
val request = SparkJobRequest("true","rapidor_realmarketing","/tmp/abc.csv","consolidated_report","none","01/02/2017","15/02/2017","","")
val sparkJobStub:CreateSparkJobBlockingStub = CreateSparkJobGrpc.blockingStub(channel)
val sparkJobStub:CreateSparkJobBlockingStub = CreateSparkJobGrpc.blockingStub(channel)
val response:SparkJobReply = sparkJobStub.createJob(request)
/* val response:SparkJobReply = sparkJobStub.createJob(request)
println("response message--------------->\t"+response.message)
println("response message--------------->\t"+response.submissionId)
val statusResponse:SparkJobReply = sparkJobStub.sparkJobStatus(SparkJobStatusReq(response.message))
val statusResponse:SparkJobStatusReply = sparkJobStub.sparkJobStatus(SparkJobStatusReq(response.submissionId))
println("Status response message--------------->\t"+statusResponse.message)
println("Status response message--------------->\t"+statusResponse.status)
val getResponse:SparkJobResponseReply = sparkJobStub.sparkJobResponse(SparkJobResponseReq(response.submissionId))
println("response message--------------->\t"+getResponse.message + "------>" +getResponse.status)
*/
val getSyncResponse:SparkJobResponseReply = sparkJobStub.createJobAndGetResult(request)
println("response message--------------->\t"+getSyncResponse.message + "------>" +getSyncResponse.status)
}
}
src/main/scala/CreateSparkJobServer.scala
import javax.inject.Inject
import com.example.protos.sparkJob.{CreateSparkJobGrpc, SparkJobReply, SparkJobRequest,SparkJobStatusReq}
import com.example.protos.sparkJob._
import com.example.protos.sparkJob.CreateSparkJobGrpc.CreateSparkJob
import models.Models.SparkJobArgs
import play.api.libs.json._ // JSON library
import play.api.libs.json._
import com.softwaremill.sttp._
import com.softwaremill.sttp.json4s._
import models.Models._
import java.io.File
import akka.actor.{Actor, ActorSystem, Props}
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration._
import scala.concurrent.{ExecutionContext, Future}
object CreateSparkJobServer extends TestServerConnect{
class CreateSparkJobServer extends CreateSparkJob {
// val system = ActorSystem("mySystem")
/*
* This method is use to create the spark job in the spark stand alone cluster and it returns the response of
* parameters ----> action:String,message:String,serverSparkVersion:String,submissionId:String,success:Boolean
*/
def createJob(req:SparkJobRequest): Future[SparkJobReply] ={
implicit val backend = HttpURLConnectionBackend()
val sparkJobArgs = SparkJobArgs(action = "CreateSubmissionRequest",appArgs = List(s"${req.debug}",s"${req.dbName}",s"${req.fileName}",s"${req.reportType}",s"${req.instanceDbs}",s"${req.startDate}",s"${req.endDate}",s"${req.reportGenerateType}",s"${req.tallyCompanyName}",s"${req.allowedCatogery}"),
val sparkJobArgs = SparkJobArgs(action = "CreateSubmissionRequest",appArgs = List(s"${req.debug}",s"${req.dbName}",s"${req.fileName}",s"${req.reportType}",s"${req.instanceDbs}",s"${req.startDate}",s"${req.endDate}",s"${req.reportGenerateType}",s"${req.tallyCompanyName}"),
appResource = System.getenv("SPARK_APP_RESOURCE"),clientSparkVersion = "1",
environmentVariables = EnvironmentVariables(SPARK_ENV_LOADED = "1", RAPIDOR_CURRENT_DB_NAME = System.getenv("RAPIDOR_CURRENT_DB_NAME"),
RAPIDOR_DB_HOSTNAME = System.getenv("RAPIDOR_DB_HOSTNAME"),RAPIDOR_DB_PORT = System.getenv("RAPIDOR_DB_PORT"),RAPIDOR_PLANX_DB_PASSWORD = System.getenv("RAPIDOR_PLANX_DB_PASSWORD"),
......@@ -40,27 +50,151 @@ object CreateSparkJobServer extends TestServerConnect{
.send()
println("----1111111---------->"+sparkResponse.body)
println("----1111111---------->"+sparkResponse)
// println("----1111111---------->"+sparkResponse.body)
// println("----1111111---------->"+sparkResponse)
println("-------------->"+req)
backend.close()
Future.successful(SparkJobReply(message = sparkResponse.body.right.get.submissionId))
Future.successful(SparkJobReply(submissionId = sparkResponse.body.right.get.submissionId))
}
def sparkJobStatus(submisionId:SparkJobStatusReq): Future[SparkJobReply] ={
/*
* This method is used to check the status of the given submissionId
* this will give the response of These
* parameters ----> action:String,driverState:String,serverSparkVersion:String,submissionId:String,success:Boolean,workerHostPort:String,workerId:String
* And the driverState are ---> `RUNNING`, `FINISHED`
*/
def sparkJobStatus(submissionId:SparkJobStatusReq): Future[SparkJobStatusReply] ={
implicit val backend = HttpURLConnectionBackend()
val sparkJobStatus = sttp.get(uri"http://192.168.1.50:6066/v1/submissions/status/${submisionId.submitionId}")
val sparkJobStatusRes = sttp.get(uri"http://192.168.1.50:6066/v1/submissions/status/${submissionId.submissionId}")
.response(asJson[StatusResponse])
.send()
println("-------status------>"+sparkJobStatusRes)
backend.close()
Future.successful(SparkJobStatusReply(status = sparkJobStatusRes.body.right.get.driverState))
}
/*
* This method is used to get the response of finished job
* and this will return the stauts is true/false and message
*/
def sparkJobResponse(resArgs:SparkJobResponseReq): Future[SparkJobResponseReply] ={
var responseData = SparkJobResponseReply(status = "",message = "")
implicit val backend = HttpURLConnectionBackend()
val sparkJobResp = sttp.get(uri"http://192.168.1.50:6066/v1/submissions/status/${resArgs.submissionId}")
.response(asJson[StatusResponse])
.send()
println("-------status------>"+sparkJobStatus)
val jobResponse = sparkJobResp.body.right.get
val file = if(resArgs.filePath.contains("/tmp")) resArgs.filePath else "/tmp/"+resArgs.filePath
val fileExist = new File(file).exists()
if(jobResponse.driverState == "FINISHED" && fileExist) {
responseData =SparkJobResponseReply(status = "true",message = "")
}else{
responseData =SparkJobResponseReply(status = "false",message = "getting error when generating the report")
}
backend.close()
Future.successful(SparkJobReply(message = sparkJobStatus.body.right.get.driverState))
Future.successful(responseData)
}
/*
* This method is used to get the synchronous method to get the all information of job and it will get the FINISHED job response
*/
def createJobAndGetResult(reqArgs:SparkJobRequest): Future[SparkJobResponseReply] ={
var sparkJobRes = SparkJobResponseReply(status = "",message = "")
implicit val backend = HttpURLConnectionBackend()
println("inside the syncronous createJobAndGetResult method")
val result:Future[SparkJobResponseReply] = Future {
val sparkJobArgs = SparkJobArgs(action = "CreateSubmissionRequest", appArgs = List(s"${reqArgs.debug}", s"${reqArgs.dbName}", s"${reqArgs.fileName}", s"${reqArgs.reportType}", s"${reqArgs.instanceDbs}", s"${reqArgs.startDate}", s"${reqArgs.endDate}", s"${reqArgs.reportGenerateType}", s"${reqArgs.tallyCompanyName}"),
appResource = System.getenv("SPARK_APP_RESOURCE"), clientSparkVersion = "1",
environmentVariables = EnvironmentVariables(SPARK_ENV_LOADED = "1", RAPIDOR_CURRENT_DB_NAME = System.getenv("RAPIDOR_CURRENT_DB_NAME"),
RAPIDOR_DB_HOSTNAME = System.getenv("RAPIDOR_DB_HOSTNAME"), RAPIDOR_DB_PORT = System.getenv("RAPIDOR_DB_PORT"), RAPIDOR_PLANX_DB_PASSWORD = System.getenv("RAPIDOR_PLANX_DB_PASSWORD"),
RAPIDOR_PLANX_DB_USERNAME = System.getenv("RAPIDOR_PLANX_DB_USERNAME")),
mainClass = "com.acelrtech.Main", sparkProperties = SparkProperties(`spark.app.name` = "MyJob",
`spark.driver.cores` = "6", `spark.eventLog.enabled` = "true", `spark.submit.deployMode` = "cluster", `spark.driver.memory` = "4G",
`spark.executor.cores` = "4", `spark.executor.memory` = "2G"))
val inputJson = Json.toJson(sparkJobArgs)
val payload = Json.stringify(inputJson)
val jobCreateResponse = sttp.contentType("application/json")
.post(uri"http://192.168.1.50:6066/v1/submissions/create")
.body(payload)
.response(asJson[ResponseJson])
.send()
var statusRes=(submissionId:String) => {
implicit val backend = HttpURLConnectionBackend()
val x = sttp.get(uri"http://192.168.1.50:6066/v1/submissions/status/${submissionId}")
.response(asJson[StatusResponse])
.send()
backend.close()
x
}
var sparkJobStatusRes: Response[StatusResponse] = statusRes(jobCreateResponse.body.right.get.submissionId)
var timeShedule = System.currentTimeMillis() + System.getenv("SHEDULE_TIME_FOR_STATUS").toLong
var currentTime = System.currentTimeMillis()
while (sparkJobStatusRes.body.right.get.driverState == "RUNNING" && currentTime <= timeShedule) {
sparkJobStatusRes = statusRes(jobCreateResponse.body.right.get.submissionId)
println(sparkJobStatusRes.body.right.get.submissionId + "---------->" + sparkJobStatusRes.body.right.get.driverState + "-------------->" + currentTime)
Thread.sleep(5000)
currentTime = System.currentTimeMillis()
}
/* val sparkJobActor = system.actorOf(Props[Test],"newActor")
implicit val ec = system.dispatcher
val act = system.scheduler.schedule(5 seconds,30 seconds, sparkJobActor,jobCreateResponse.body.right.get.submissionId)
act*/
val jobResponse = sparkJobStatusRes.body.right.get
val file = if(reqArgs.fileName.contains("/tmp")) reqArgs.fileName else "/tmp/"+reqArgs.fileName
val fileExist = new File(file).exists()
if(jobResponse.driverState == "FINISHED" && fileExist) {
sparkJobRes =SparkJobResponseReply(status = "true",message = "")
}else{
sparkJobRes =SparkJobResponseReply(status = "false",message = "getting error when generating the report")
}
backend.close()
println("---Finished--->")
sparkJobRes
}
//Future.successful(sparkJobRes)
result
}
}
/*class Test extends Actor{
def receive={
case s:String=> sender() ! new CreateSparkJobServer().statusRes(s)
}
}*/
def main(args:Array[String]): Unit ={
val ssd = CreateSparkJobGrpc.bindService(new CreateSparkJobServer, ExecutionContext.global)
......
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!