Skip to content

Girish A Nandolkar / grpc-project

Go to a project
Commit d878752e by Girish A Nandolkar
Options

deleted some unused files

1 parent c7e177b0
Expand all
Inline Side-by-side
Showing with 19 additions and 499 deletions
build.sbt deleted 100644 → 0
name := "newScalaExp"
version := "0.1"
scalaVersion := "2.12.4"
//resolvers += "theatr.us" at "http://repo.theatr.us"
PB.targets in Compile := Seq(
scalapb.gen() -> (sourceManaged in Compile).value
)
libraryDependencies ++= Seq(
"com.typesafe.play" %% "play-ws" % "2.6.7",
"com.softwaremill.sttp" %% "core" % "1.1.0",
"com.softwaremill.sttp" %% "json4s" % "1.1.0",
"com.typesafe.akka" %% "akka-actor" % "2.4.8",
// "us.theatr" %% "akka-quartz" % "0.3.0",
"io.grpc" % "grpc-netty" % com.trueaccord.scalapb.compiler.Version.grpcJavaVersion,
"com.trueaccord.scalapb" %% "scalapb-runtime" % com.trueaccord.scalapb.compiler.Version.scalapbVersion % "protobuf",
"com.trueaccord.scalapb" %% "scalapb-runtime-grpc" % com.trueaccord.scalapb.compiler.Version.scalapbVersion
)
newScalaExp/.idea/workspace.xml
...@@ -75,8 +75,8 @@ ...@@ -75,8 +75,8 @@
<file leaf-file-name="CreateSparkJobServer.scala" pinned="false" current-in-tab="true"> <file leaf-file-name="CreateSparkJobServer.scala" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/src/main/scala/CreateSparkJobServer.scala"> <entry file="file://$PROJECT_DIR$/src/main/scala/CreateSparkJobServer.scala">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="437"> <state relative-caret-position="-92">
<caret line="91" column="27" lean-forward="false" selection-start-line="91" selection-start-column="20" selection-end-line="91" selection-end-column="27" /> <caret line="77" column="0" lean-forward="true" selection-start-line="91" selection-start-column="20" selection-end-line="91" selection-end-column="27" />
<folding> <folding>
<element signature="e#3#39#0" expanded="true" /> <element signature="e#3#39#0" expanded="true" />
</folding> </folding>
...@@ -177,38 +177,6 @@ ...@@ -177,38 +177,6 @@
<item name="newScalaExp" type="b2602c69:ProjectViewProjectNode" /> <item name="newScalaExp" type="b2602c69:ProjectViewProjectNode" />
<item name="newScalaExp" type="462c0819:PsiDirectoryNode" /> <item name="newScalaExp" type="462c0819:PsiDirectoryNode" />
</path> </path>
<path>
<item name="newScalaExp" type="b2602c69:ProjectViewProjectNode" />
<item name="newScalaExp" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="newScalaExp" type="b2602c69:ProjectViewProjectNode" />
<item name="newScalaExp" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="newScalaExp" type="b2602c69:ProjectViewProjectNode" />
<item name="newScalaExp" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="protobuf" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="newScalaExp" type="b2602c69:ProjectViewProjectNode" />
<item name="newScalaExp" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="pythonClient" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="newScalaExp" type="b2602c69:ProjectViewProjectNode" />
<item name="newScalaExp" type="462c0819:PsiDirectoryNode" />
<item name="src" type="462c0819:PsiDirectoryNode" />
<item name="main" type="462c0819:PsiDirectoryNode" />
<item name="scala" type="462c0819:PsiDirectoryNode" />
</path>
</expand> </expand>
<select /> <select />
</subPane> </subPane>
...@@ -454,12 +422,12 @@ ...@@ -454,12 +422,12 @@
<window_info id="Capture Analysis" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" /> <window_info id="Capture Analysis" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" /> <window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" />
<window_info id="Maven Projects" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" /> <window_info id="Maven Projects" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.634555" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" /> <window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.634555" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="8" side_tool="false" content_ui="tabs" /> <window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="8" side_tool="false" content_ui="tabs" />
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="10" side_tool="false" content_ui="tabs" /> <window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="10" side_tool="false" content_ui="tabs" />
<window_info id="Capture Tool" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" /> <window_info id="Capture Tool" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" /> <window_info id="Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.18629077" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" /> <window_info id="Project" active="true" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.18629077" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="sbt-shell-toolwindow" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="11" side_tool="false" content_ui="tabs" /> <window_info id="sbt-shell-toolwindow" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="11" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" /> <window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" /> <window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
...@@ -865,8 +833,8 @@ ...@@ -865,8 +833,8 @@
</entry> </entry>
<entry file="file://$PROJECT_DIR$/src/main/scala/CreateSparkJobServer.scala"> <entry file="file://$PROJECT_DIR$/src/main/scala/CreateSparkJobServer.scala">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="437"> <state relative-caret-position="-92">
<caret line="91" column="27" lean-forward="false" selection-start-line="91" selection-start-column="20" selection-end-line="91" selection-end-column="27" /> <caret line="77" column="0" lean-forward="true" selection-start-line="91" selection-start-column="20" selection-end-line="91" selection-end-column="27" />
<folding> <folding>
<element signature="e#3#39#0" expanded="true" /> <element signature="e#3#39#0" expanded="true" />
</folding> </folding>
......
project/build.properties deleted 100644 → 0
sbt.version = 0.13.16
\ No newline at end of file
project/plugins.sbt deleted 100644 → 0
addSbtPlugin("com.thesamet" % "sbt-protoc" % "0.99.13")
libraryDependencies += "com.trueaccord.scalapb" %% "compilerplugin" % "0.6.7"
\ No newline at end of file
python_code/pythonGrpc/grpc1/rapidorSparkReportPythonClient.py
...@@ -8,7 +8,7 @@ import rapidorSparkReport_pb2_grpc ...@@ -8,7 +8,7 @@ import rapidorSparkReport_pb2_grpc
channel = grpc.insecure_channel('localhost:50051') channel = grpc.insecure_channel('localhost:50051')
stub = rapidorSparkReport_pb2_grpc.CreateSparkJobStub(channel) stub = rapidorSparkReport_pb2_grpc.CreateSparkJobStub(channel)
def run(): def runSynchronus():
reqData =rapidorSparkReport_pb2.SparkJobRequest(debug="true",dbName="rapidor_realmarketing",fileName="/tmp/abc.csv",reportType="consolidated_report",instanceDbs="none",startDate="01/02/2017",endDate="15/02/2017",reportGenerateType="",tallyCompanyName="") reqData =rapidorSparkReport_pb2.SparkJobRequest(debug="true",dbName="rapidor_realmarketing",fileName="/tmp/abc.csv",reportType="consolidated_report",instanceDbs="none",startDate="01/02/2017",endDate="15/02/2017",reportGenerateType="",tallyCompanyName="")
response = stub.CreateJobAndGetResult(reqData) response = stub.CreateJobAndGetResult(reqData)
...@@ -34,5 +34,5 @@ def createSparkJob(): ...@@ -34,5 +34,5 @@ def createSparkJob():
if __name__ == '__main__': if __name__ == '__main__':
#run() runSynchronus()
createSparkJob() #createSparkJob()
\ No newline at end of file \ No newline at end of file
python_code/pythonGrpc/pythonGrpc.ipr
...@@ -11,7 +11,7 @@ ...@@ -11,7 +11,7 @@
<module fileurl="file://$PROJECT_DIR$/pythonGrpc.iml" filepath="$PROJECT_DIR$/pythonGrpc.iml" /> <module fileurl="file://$PROJECT_DIR$/pythonGrpc.iml" filepath="$PROJECT_DIR$/pythonGrpc.iml" />
</modules> </modules>
</component> </component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_9" default="false" project-jdk-name="Python 2.7.12 virtualenv at ~/mywork/grpcSparkApi/pythonGrpc/venv" project-jdk-type="Python SDK"> <component name="ProjectRootManager" version="2" languageLevel="JDK_1_9" default="false" project-jdk-name="Python 2.7.12 virtualenv at ~/mywork/grpcSparkApi/grpc-project/python_code/pythonGrpc/venv" project-jdk-type="Python SDK">
<output url="file://$PROJECT_DIR$/out" /> <output url="file://$PROJECT_DIR$/out" />
</component> </component>
<component name="ScalaSbtSettings"> <component name="ScalaSbtSettings">
......
python_code/pythonGrpc/pythonGrpc.iws
...@@ -35,8 +35,8 @@ ...@@ -35,8 +35,8 @@
<file leaf-file-name="rapidorSparkReportPythonClient.py" pinned="false" current-in-tab="true"> <file leaf-file-name="rapidorSparkReportPythonClient.py" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/grpc1/rapidorSparkReportPythonClient.py"> <entry file="file://$PROJECT_DIR$/grpc1/rapidorSparkReportPythonClient.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="184"> <state relative-caret-position="92">
<caret line="8" column="50" lean-forward="false" selection-start-line="8" selection-start-column="50" selection-end-line="8" selection-end-column="50" /> <caret line="10" column="20" lean-forward="true" selection-start-line="10" selection-start-column="20" selection-end-line="10" selection-end-column="20" />
<folding> <folding>
<element signature="e#1#38#0" expanded="true" /> <element signature="e#1#38#0" expanded="true" />
</folding> </folding>
...@@ -160,7 +160,7 @@ ...@@ -160,7 +160,7 @@
<property name="project.structure.proportion" value="0.15" /> <property name="project.structure.proportion" value="0.15" />
<property name="project.structure.side.proportion" value="0.2" /> <property name="project.structure.side.proportion" value="0.2" />
<property name="settings.editor.selected.configurable" value="org.jetbrains.plugins.hocon.settings.HoconProjectSettingsConfigurable" /> <property name="settings.editor.selected.configurable" value="org.jetbrains.plugins.hocon.settings.HoconProjectSettingsConfigurable" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" /> <property name="last_opened_file_path" value="$PROJECT_DIR$/venv/bin/python" />
</component> </component>
<component name="RecentsManager"> <component name="RecentsManager">
<key name="MoveFile.RECENT_KEYS"> <key name="MoveFile.RECENT_KEYS">
...@@ -391,12 +391,12 @@ ...@@ -391,12 +391,12 @@
<window_info id="Capture Analysis" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" /> <window_info id="Capture Analysis" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" /> <window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" />
<window_info id="Maven Projects" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" /> <window_info id="Maven Projects" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.3291536" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" /> <window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.3291536" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="false" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" /> <window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="false" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" /> <window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Capture Tool" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" /> <window_info id="Capture Tool" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" /> <window_info id="Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Project" active="true" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.25262576" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" /> <window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.32559425" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" /> <window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" /> <window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="UI Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" /> <window_info id="UI Designer" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
...@@ -587,8 +587,8 @@ ...@@ -587,8 +587,8 @@
</entry> </entry>
<entry file="file://$PROJECT_DIR$/grpc1/rapidorSparkReportPythonClient.py"> <entry file="file://$PROJECT_DIR$/grpc1/rapidorSparkReportPythonClient.py">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="184"> <state relative-caret-position="92">
<caret line="8" column="50" lean-forward="false" selection-start-line="8" selection-start-column="50" selection-end-line="8" selection-end-column="50" /> <caret line="10" column="20" lean-forward="true" selection-start-line="10" selection-start-column="20" selection-end-line="10" selection-end-column="20" />
<folding> <folding>
<element signature="e#1#38#0" expanded="true" /> <element signature="e#1#38#0" expanded="true" />
</folding> </folding>
...@@ -636,7 +636,7 @@ ...@@ -636,7 +636,7 @@
</state> </state>
<state key="JdkListConfigurable.UI"> <state key="JdkListConfigurable.UI">
<settings> <settings>
<last-edited>1.8</last-edited> <last-edited>Python 2.7.12 virtualenv at ~/mywork/grpcSparkApi/pythonGrpc/venv</last-edited>
<splitter-proportions> <splitter-proportions>
<option name="proportions"> <option name="proportions">
<list> <list>
......
src/main/protobuf/hello.proto deleted 100644 → 0
syntax = "proto3";
package com.example.protos;
// The greeting service definition.
service Greeter {
// Sends a greeting
rpc SayHello (HelloRequest) returns (HelloReply) {}
}
// The request message containing the user's name.
message HelloRequest {
string name = 1;
}
// The response message containing the greetings
message HelloReply {
string message = 1;
}
\ No newline at end of file
src/main/protobuf/sparkJob.proto deleted 100644 → 0
syntax = "proto3";
package com.example.protos;
service CreateSparkJob {
rpc CreateJob (SparkJobRequest) returns (SparkJobReply) {}
rpc SparkJobStatus (SparkJobStatusReq) returns (SparkJobStatusReply) {}
rpc SparkJobResponse (SparkJobResponseReq) returns (SparkJobResponseReply) {}
rpc CreateJobAndGetResult(SparkJobRequest) returns (SparkJobResponseReply) {}
}
// The request message containing the debug, dbName, filename, reporttype, instanceDbs, startDate,endDate, reportGenerateType,tallyCompanyName,allowCatogery.
message SparkJobRequest {
string debug = 1;
string dbName = 2;
string fileName = 3;
string reportType = 4;
string instanceDbs = 5;
string startDate = 6;
string endDate = 7;
string reportGenerateType = 8;
string tallyCompanyName = 9;
}
// The response message containing the greetings
message SparkJobReply {
string submissionId = 1;
}
message SparkJobStatusReq{
string submissionId = 1;
}
message SparkJobStatusReply{
string status = 1;
}
message SparkJobResponseReq{
string submissionId = 1;
string filePath = 2;
}
message SparkJobResponseReply{
string status = 1;
string message = 2;
}
src/main/scala/CreateSparkJobClient.scala deleted 100644 → 0
import com.example.protos.sparkJob.CreateSparkJobGrpc.{CreateSparkJob, CreateSparkJobBlockingStub}
import com.example.protos.sparkJob._
import io.grpc.ManagedChannelBuilder
object CreateSparkJobClient {
def main(args:Array[String]): Unit ={
val channel = ManagedChannelBuilder.forAddress("localhost",50051)
.usePlaintext(true).build()
val request = SparkJobRequest("true","rapidor_realmarketing","/tmp/abc.csv","consolidated_report","none","01/02/2017","15/02/2017","","")
val sparkJobStub:CreateSparkJobBlockingStub = CreateSparkJobGrpc.blockingStub(channel)
/* val response:SparkJobReply = sparkJobStub.createJob(request)
println("response message--------------->\t"+response.submissionId)
val statusResponse:SparkJobStatusReply = sparkJobStub.sparkJobStatus(SparkJobStatusReq(response.submissionId))
println("Status response message--------------->\t"+statusResponse.status)
val getResponse:SparkJobResponseReply = sparkJobStub.sparkJobResponse(SparkJobResponseReq(response.submissionId))
println("response message--------------->\t"+getResponse.message + "------>" +getResponse.status)
*/
val getSyncResponse:SparkJobResponseReply = sparkJobStub.createJobAndGetResult(request)
println("response message--------------->\t"+getSyncResponse.message + "------>" +getSyncResponse.status)
}
}
src/main/scala/CreateSparkJobServer.scala deleted 100644 → 0
import com.example.protos.sparkJob._
import com.example.protos.sparkJob.CreateSparkJobGrpc.CreateSparkJob
import models.Models.SparkJobArgs
import play.api.libs.json._
import com.softwaremill.sttp._
import com.softwaremill.sttp.json4s._
import models.Models._
import java.io.File
import akka.actor.{Actor, ActorSystem, Props}
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration._
import scala.concurrent.{ExecutionContext, Future}
object CreateSparkJobServer extends TestServerConnect{
class CreateSparkJobServer extends CreateSparkJob {
// val system = ActorSystem("mySystem")
/*
* This method is use to create the spark job in the spark stand alone cluster and it returns the response of
* parameters ----> action:String,message:String,serverSparkVersion:String,submissionId:String,success:Boolean
*/
def createJob(req:SparkJobRequest): Future[SparkJobReply] ={
implicit val backend = HttpURLConnectionBackend()
val sparkJobArgs = SparkJobArgs(action = "CreateSubmissionRequest",appArgs = List(s"${req.debug}",s"${req.dbName}",s"${req.fileName}",s"${req.reportType}",s"${req.instanceDbs}",s"${req.startDate}",s"${req.endDate}",s"${req.reportGenerateType}",s"${req.tallyCompanyName}"),
appResource = System.getenv("SPARK_APP_RESOURCE"),clientSparkVersion = "1",
environmentVariables = EnvironmentVariables(SPARK_ENV_LOADED = "1", RAPIDOR_CURRENT_DB_NAME = System.getenv("RAPIDOR_CURRENT_DB_NAME"),
RAPIDOR_DB_HOSTNAME = System.getenv("RAPIDOR_DB_HOSTNAME"),RAPIDOR_DB_PORT = System.getenv("RAPIDOR_DB_PORT"),RAPIDOR_PLANX_DB_PASSWORD = System.getenv("RAPIDOR_PLANX_DB_PASSWORD"),
RAPIDOR_PLANX_DB_USERNAME = System.getenv("RAPIDOR_PLANX_DB_USERNAME")),
mainClass = "com.acelrtech.Main",sparkProperties = SparkProperties(`spark.app.name` = "MyJob",
`spark.driver.cores` = "6",`spark.eventLog.enabled` = "true",`spark.submit.deployMode` = "cluster", `spark.driver.memory` = "4G",
`spark.executor.cores` = "4",`spark.executor.memory` = "2G"))
val inputJson = Json.toJson(sparkJobArgs)
val payload = Json.stringify(inputJson)
val sparkResponse = sttp.contentType("application/json")
.post(uri"http://192.168.1.50:6066/v1/submissions/create")
.body(payload)
.response(asJson[ResponseJson])
.send()
// println("----1111111---------->"+sparkResponse.body)
// println("----1111111---------->"+sparkResponse)
println("-------------->"+req)
backend.close()
Future.successful(SparkJobReply(submissionId = sparkResponse.body.right.get.submissionId))
}
/*
* This method is used to check the status of the given submissionId
* this will give the response of These
* parameters ----> action:String,driverState:String,serverSparkVersion:String,submissionId:String,success:Boolean,workerHostPort:String,workerId:String
* And the driverState are ---> `RUNNING`, `FINISHED`
*/
def sparkJobStatus(submissionId:SparkJobStatusReq): Future[SparkJobStatusReply] ={
implicit val backend = HttpURLConnectionBackend()
val sparkJobStatusRes = sttp.get(uri"http://192.168.1.50:6066/v1/submissions/status/${submissionId.submissionId}")
.response(asJson[StatusResponse])
.send()
println("-------status------>"+sparkJobStatusRes)
backend.close()
Future.successful(SparkJobStatusReply(status = sparkJobStatusRes.body.right.get.driverState))
}
/*
* This method is used to get the response of finished job
* and this will return the stauts is true/false and message
*/
def sparkJobResponse(resArgs:SparkJobResponseReq): Future[SparkJobResponseReply] ={
var responseData = SparkJobResponseReply(status = "",message = "")
implicit val backend = HttpURLConnectionBackend()
val sparkJobResp = sttp.get(uri"http://192.168.1.50:6066/v1/submissions/status/${resArgs.submissionId}")
.response(asJson[StatusResponse])
.send()
val jobResponse = sparkJobResp.body.right.get
val file = if(resArgs.filePath.contains("/tmp")) resArgs.filePath else "/tmp/"+resArgs.filePath
val fileExist = new File(file).exists()
if(jobResponse.driverState == "FINISHED" && fileExist) {
responseData =SparkJobResponseReply(status = "true",message = "")
}else{
responseData =SparkJobResponseReply(status = "false",message = "getting error when generating the report")
}
backend.close()
Future.successful(responseData)
}
/*
* This method is used to get the synchronous method to get the all information of job and it will get the FINISHED job response
*/
def createJobAndGetResult(reqArgs:SparkJobRequest): Future[SparkJobResponseReply] ={
var sparkJobRes = SparkJobResponseReply(status = "",message = "")
implicit val backend = HttpURLConnectionBackend()
println("inside the syncronous createJobAndGetResult method")
val result:Future[SparkJobResponseReply] = Future {
val sparkJobArgs = SparkJobArgs(action = "CreateSubmissionRequest", appArgs = List(s"${reqArgs.debug}", s"${reqArgs.dbName}", s"${reqArgs.fileName}", s"${reqArgs.reportType}", s"${reqArgs.instanceDbs}", s"${reqArgs.startDate}", s"${reqArgs.endDate}", s"${reqArgs.reportGenerateType}", s"${reqArgs.tallyCompanyName}"),
appResource = System.getenv("SPARK_APP_RESOURCE"), clientSparkVersion = "1",
environmentVariables = EnvironmentVariables(SPARK_ENV_LOADED = "1", RAPIDOR_CURRENT_DB_NAME = System.getenv("RAPIDOR_CURRENT_DB_NAME"),
RAPIDOR_DB_HOSTNAME = System.getenv("RAPIDOR_DB_HOSTNAME"), RAPIDOR_DB_PORT = System.getenv("RAPIDOR_DB_PORT"), RAPIDOR_PLANX_DB_PASSWORD = System.getenv("RAPIDOR_PLANX_DB_PASSWORD"),
RAPIDOR_PLANX_DB_USERNAME = System.getenv("RAPIDOR_PLANX_DB_USERNAME")),
mainClass = "com.acelrtech.Main", sparkProperties = SparkProperties(`spark.app.name` = "MyJob",
`spark.driver.cores` = "6", `spark.eventLog.enabled` = "true", `spark.submit.deployMode` = "cluster", `spark.driver.memory` = "4G",
`spark.executor.cores` = "4", `spark.executor.memory` = "2G"))
val inputJson = Json.toJson(sparkJobArgs)
val payload = Json.stringify(inputJson)
val jobCreateResponse = sttp.contentType("application/json")
.post(uri"http://192.168.1.50:6066/v1/submissions/create")
.body(payload)
.response(asJson[ResponseJson])
.send()
var statusRes=(submissionId:String) => {
implicit val backend = HttpURLConnectionBackend()
val x = sttp.get(uri"http://192.168.1.50:6066/v1/submissions/status/${submissionId}")
.response(asJson[StatusResponse])
.send()
backend.close()
x
}
var sparkJobStatusRes: Response[StatusResponse] = statusRes(jobCreateResponse.body.right.get.submissionId)
var timeShedule = System.currentTimeMillis() + System.getenv("SHEDULE_TIME_FOR_STATUS").toLong
var currentTime = System.currentTimeMillis()
while (sparkJobStatusRes.body.right.get.driverState == "RUNNING" && currentTime <= timeShedule) {
sparkJobStatusRes = statusRes(jobCreateResponse.body.right.get.submissionId)
println(sparkJobStatusRes.body.right.get.submissionId + "---------->" + sparkJobStatusRes.body.right.get.driverState + "-------------->" + currentTime)
Thread.sleep(5000)
currentTime = System.currentTimeMillis()
}
/* val sparkJobActor = system.actorOf(Props[Test],"newActor")
implicit val ec = system.dispatcher
val act = system.scheduler.schedule(5 seconds,30 seconds, sparkJobActor,jobCreateResponse.body.right.get.submissionId)
act*/
val jobResponse = sparkJobStatusRes.body.right.get
val file = if(reqArgs.fileName.contains("/tmp")) reqArgs.fileName else "/tmp/"+reqArgs.fileName
val fileExist = new File(file).exists()
if(jobResponse.driverState == "FINISHED" && fileExist) {
sparkJobRes =SparkJobResponseReply(status = "true",message = "")
}else{
sparkJobRes =SparkJobResponseReply(status = "false",message = "getting error when generating the report")
}
backend.close()
println("---Finished--->")
sparkJobRes
}
//Future.successful(sparkJobRes)
result
}
}
/*class Test extends Actor{
def receive={
case s:String=> sender() ! new CreateSparkJobServer().statusRes(s)
}
}*/
def main(args:Array[String]): Unit ={
val ssd = CreateSparkJobGrpc.bindService(new CreateSparkJobServer, ExecutionContext.global)
runServer(ssd)
}
}
src/main/scala/Models.scala deleted 100644 → 0
package models
import play.api.libs.json.Json
object Models {
case class EnvironmentVariables(SPARK_ENV_LOADED:String, RAPIDOR_CURRENT_DB_NAME:String,RAPIDOR_DB_HOSTNAME:String,RAPIDOR_DB_PORT:String,
RAPIDOR_PLANX_DB_PASSWORD:String,RAPIDOR_PLANX_DB_USERNAME:String)
object EnvironmentVariables {
implicit val format1 = Json.format[EnvironmentVariables]
}
case class SparkProperties(`spark.app.name`:String,`spark.eventLog.enabled`:String,`spark.driver.memory`:String,`spark.driver.cores`:String,
`spark.executor.memory`:String,`spark.executor.cores`:String,`spark.submit.deployMode`:String)
object SparkProperties {
implicit val format2 = Json.format[SparkProperties]
}
import EnvironmentVariables.format1
import SparkProperties.format2
case class SparkJobArgs(action:String,appArgs:List[String],appResource:String,clientSparkVersion:String,
environmentVariables:EnvironmentVariables,mainClass:String,
sparkProperties:SparkProperties)
object SparkJobArgs {
implicit val format3 = Json.format[SparkJobArgs]
}
import SparkJobArgs._
case class ResponseJson(action:String,message:String,serverSparkVersion:String,submissionId:String,success:Boolean)
object ResponseJson{
implicit val format4 = Json.format[ResponseJson]
}
import ResponseJson._
case class StatusResponse(action:String,driverState:String,serverSparkVersion:String,submissionId:String,success:Boolean,workerHostPort:String,workerId:String)
object StatusResponse{
implicit val formatStatus = Json.format[StatusResponse]
}
}
src/main/scala/TestClient.scala deleted 100644 → 0
import com.example.protos.hello.GreeterGrpc.{Greeter, GreeterBlockingStub, GreeterStub}
import com.example.protos.hello.{GreeterGrpc, HelloReply, HelloRequest}
import io.grpc.ManagedChannelBuilder
import scala.concurrent.Future
import scala.concurrent.ExecutionContext.Implicits.global
object TestClient {
def main(args:Array[String]): Unit ={
val channel = ManagedChannelBuilder.forAddress("localhost",50051)
.usePlaintext(true).build()
val request = HelloRequest("Good Morning Girish")
/* val helloWorld:GreeterStub = GreeterGrpc.stub(channel)
val greF:Future[HelloReply]= helloWorld.sayHello(request)
greF.foreach(res => println("Response message---------->"+{res.message}))*/
val blockingStub: GreeterBlockingStub = GreeterGrpc.blockingStub(channel)
val greeting: HelloReply = blockingStub.sayHello(request)
println(s"SYNC(BLOCKING) RESULT: ${greeting.message}")
}
}
src/main/scala/TestServer.scala deleted 100644 → 0
import com.example.protos.hello.GreeterGrpc.Greeter
import com.example.protos.hello.{GreeterGrpc, HelloReply, HelloRequest}
import scala.concurrent.{ExecutionContext, Future}
object TestServer extends TestServerConnect{
class TestServer extends Greeter{
def sayHello(req:HelloRequest): Future[HelloReply] ={
println("-------------->"+req)
Future.successful(HelloReply(message = s"Hello ${req.name}"))
}
}
def main(args:Array[String]): Unit ={
val ssd = GreeterGrpc.bindService(new TestServer, ExecutionContext.global)
runServer(ssd)
}
}
src/main/scala/TestServerConnect.scala deleted 100644 → 0
import io.grpc.{ServerBuilder, ServerServiceDefinition}
trait TestServerConnect {
/**
* Just for demo purposes
*/
def runServer(ssd: ServerServiceDefinition): Unit = {
val server = ServerBuilder
.forPort(50051)
.addService(ssd)
.build
.start
// make sure our server is stopped when jvm is shut down
Runtime.getRuntime.addShutdownHook(new Thread() {
override def run(): Unit = server.shutdown()
})
server.awaitTermination()
}
}
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!