Skip to content

Guest User / general

Go to a project
Commit bc4b108a by hdjay01
Options

Github website scrapper made by hindu singh

1 parent aaf1d7f1
Inline Side-by-side
Showing with 927 additions and 1 deletions
.classpath 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="target/classes" path="src/main/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.5">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="output" path="target/classes"/>
</classpath>
.project 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>blog</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
</natures>
</projectDescription>
.settings/org.eclipse.jdt.core.prefs 0 → 100644
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5
org.eclipse.jdt.core.compiler.compliance=1.5
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.source=1.5
.settings/org.eclipse.m2e.core.prefs 0 → 100644
activeProfiles=
eclipse.preferences.version=1
resolveWorkspaceProjects=true
version=1
README.md
git clone http://pawan@projects.rapidor.co/pawan/general.git
Made by - Hindu Singh
To run:
go to target directory:
Run the below command to execute the jar file to run the
-> java -jar webscraper-0.0.1-jar-with-dependencies.jar
Note: *JavaRuntimeEnvironment is required to be installed on the system to run this.
pom.xml 0 → 100644
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>hdjay</groupId>
<artifactId>webscraper</artifactId>
<version>0.0.1</version>
<name>WebScraper</name>
<build>
<plugins>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<addClasspath>true</addClasspath>
<classpathPrefix>lib/</classpathPrefix>
<mainClass>Main.Scraper</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
<plugin>
<!-- Build an executable JAR -->
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
<configuration>
<archive>
<manifest>
<addClasspath>true</addClasspath>
<classpathPrefix>lib/</classpathPrefix>
<mainClass>Main.Scraper</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>net.sourceforge.htmlunit</groupId>
<artifactId>htmlunit</artifactId>
<version>2.19</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.7.0</version>
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
<artifactId>selenium-java</artifactId>
<version>2.53.1</version>
</dependency>
<dependency>
<groupId>com.github.detro</groupId>
<artifactId>phantomjsdriver</artifactId>
<version>1.2.0</version>
</dependency>
</dependencies>
</project>
\ No newline at end of file
src/main/java/Main/Item.java 0 → 100644
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package Main;
/**
*
* @author Hindu Singh
*/
import java.math.BigDecimal;
public class Item {
private String title ;
private String url;
private String desc ;
private String rating ;
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
public String getUrl() {
return url;
}
public void setUrl(String url){
this.url = url;
}
public String getDesc() {
return desc;
}
public void setDesc(String desc) {
this.desc = desc;
}
public String getRating() {
return rating;
}
public void setRating(String rating) {
this.rating = rating;
}
}
src/main/java/Main/Language.java 0 → 100644
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package Main;
/**
*
* @author Hindu Singh
*/
public class Language {
private String name ;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
}
src/main/java/Main/Scraper.form 0 → 100644
<?xml version="1.0" encoding="UTF-8" ?>
<Form version="1.5" maxVersion="1.9" type="org.netbeans.modules.form.forminfo.JFrameFormInfo">
<Properties>
<Property name="defaultCloseOperation" type="int" value="3"/>
</Properties>
<SyntheticProperties>
<SyntheticProperty name="formSizePolicy" type="int" value="1"/>
<SyntheticProperty name="generateCenter" type="boolean" value="false"/>
</SyntheticProperties>
<AuxValues>
<AuxValue name="FormSettings_autoResourcing" type="java.lang.Integer" value="0"/>
<AuxValue name="FormSettings_autoSetComponentName" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_generateFQN" type="java.lang.Boolean" value="true"/>
<AuxValue name="FormSettings_generateMnemonicsCode" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_i18nAutoMode" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_layoutCodeTarget" type="java.lang.Integer" value="1"/>
<AuxValue name="FormSettings_listenerGenerationStyle" type="java.lang.Integer" value="0"/>
<AuxValue name="FormSettings_variablesLocal" type="java.lang.Boolean" value="false"/>
<AuxValue name="FormSettings_variablesModifier" type="java.lang.Integer" value="2"/>
</AuxValues>
<Layout>
<DimensionLayout dim="0">
<Group type="103" groupAlignment="0" attributes="0">
<Component id="jPanel1" alignment="0" max="32767" attributes="0"/>
</Group>
</DimensionLayout>
<DimensionLayout dim="1">
<Group type="103" groupAlignment="0" attributes="0">
<Component id="jPanel1" alignment="0" max="32767" attributes="0"/>
</Group>
</DimensionLayout>
</Layout>
<SubComponents>
<Container class="javax.swing.JPanel" name="jPanel1">
<Properties>
<Property name="background" type="java.awt.Color" editor="org.netbeans.beaninfo.editors.ColorEditor">
<Color blue="99" green="99" red="99" type="rgb"/>
</Property>
</Properties>
<Layout>
<DimensionLayout dim="0">
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" attributes="0">
<Group type="103" groupAlignment="0" attributes="0">
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" attributes="0">
<EmptySpace min="-2" pref="315" max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
<Component id="jLabel1" min="-2" max="-2" attributes="0"/>
<Group type="102" alignment="0" attributes="0">
<Component id="searchText" min="-2" pref="155" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" max="-2" attributes="0">
<Component id="clear" max="32767" attributes="0"/>
<Component id="jButton1" max="32767" attributes="0"/>
</Group>
</Group>
</Group>
<EmptySpace min="-2" pref="137" max="-2" attributes="0"/>
</Group>
<Group type="102" alignment="1" attributes="0">
<EmptySpace min="20" pref="20" max="-2" attributes="0"/>
<Component id="jScrollPane3" max="32767" attributes="0"/>
</Group>
</Group>
<Group type="102" alignment="0" attributes="0">
<EmptySpace min="-2" pref="96" max="-2" attributes="0"/>
<Component id="searchstatus" min="-2" pref="485" max="-2" attributes="0"/>
</Group>
</Group>
<EmptySpace min="-2" pref="15" max="-2" attributes="0"/>
<Component id="jScrollPane1" pref="193" max="32767" attributes="0"/>
<EmptySpace min="-2" max="-2" attributes="0"/>
</Group>
<Group type="102" alignment="0" attributes="0">
<EmptySpace min="-2" pref="249" max="-2" attributes="0"/>
<Component id="previous" min="-2" max="-2" attributes="0"/>
<EmptySpace type="separate" max="-2" attributes="0"/>
<Component id="pageNo" min="-2" pref="27" max="-2" attributes="0"/>
<EmptySpace max="-2" attributes="0"/>
<Component id="next" min="-2" pref="70" max="-2" attributes="0"/>
<EmptySpace max="32767" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
<DimensionLayout dim="1">
<Group type="103" groupAlignment="0" attributes="0">
<Group type="102" alignment="0" attributes="0">
<EmptySpace min="-2" max="-2" attributes="0"/>
<Component id="jLabel1" min="-2" max="-2" attributes="0"/>
<EmptySpace type="unrelated" min="-2" max="-2" attributes="0"/>
<Group type="103" groupAlignment="3" attributes="0">
<Component id="searchText" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="jButton1" alignment="3" min="-2" max="-2" attributes="0"/>
</Group>
<EmptySpace type="unrelated" min="-2" max="-2" attributes="0"/>
<Component id="clear" min="-2" max="-2" attributes="0"/>
<EmptySpace min="-2" pref="36" max="-2" attributes="0"/>
<Component id="searchstatus" min="-2" pref="22" max="-2" attributes="0"/>
<EmptySpace min="-2" max="-2" attributes="0"/>
<Group type="103" groupAlignment="0" attributes="0">
<Component id="jScrollPane1" pref="344" max="32767" attributes="0"/>
<Component id="jScrollPane3" pref="0" max="32767" attributes="0"/>
</Group>
<EmptySpace type="unrelated" min="-2" max="-2" attributes="0"/>
<Group type="103" groupAlignment="3" attributes="0">
<Component id="previous" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="pageNo" alignment="3" min="-2" max="-2" attributes="0"/>
<Component id="next" alignment="3" min="-2" max="-2" attributes="0"/>
</Group>
<EmptySpace min="-2" max="-2" attributes="0"/>
</Group>
</Group>
</DimensionLayout>
</Layout>
<SubComponents>
<Component class="javax.swing.JLabel" name="jLabel1">
<Properties>
<Property name="font" type="java.awt.Font" editor="org.netbeans.beaninfo.editors.FontEditor">
<Font name="Tahoma" size="14" style="1"/>
</Property>
<Property name="foreground" type="java.awt.Color" editor="org.netbeans.beaninfo.editors.ColorEditor">
<Color blue="ff" green="ff" red="ff" type="rgb"/>
</Property>
<Property name="text" type="java.lang.String" value="WebScapper For GitHub By Hindu Singh"/>
</Properties>
</Component>
<Component class="javax.swing.JTextField" name="searchText">
<Properties>
<Property name="toolTipText" type="java.lang.String" value="search"/>
</Properties>
</Component>
<Component class="javax.swing.JButton" name="jButton1">
<Properties>
<Property name="text" type="java.lang.String" value="Search"/>
</Properties>
<Events>
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="jButton1ActionPerformed"/>
</Events>
</Component>
<Component class="javax.swing.JButton" name="previous">
<Properties>
<Property name="text" type="java.lang.String" value="Previous"/>
</Properties>
<Events>
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="previousActionPerformed"/>
</Events>
</Component>
<Component class="javax.swing.JLabel" name="pageNo">
<Properties>
<Property name="text" type="java.lang.String" value="1"/>
</Properties>
</Component>
<Component class="javax.swing.JButton" name="next">
<Properties>
<Property name="text" type="java.lang.String" value="Next"/>
</Properties>
<Events>
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="nextActionPerformed"/>
</Events>
</Component>
<Component class="javax.swing.JButton" name="clear">
<Properties>
<Property name="text" type="java.lang.String" value="Clear"/>
</Properties>
<Events>
<EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="clearActionPerformed"/>
</Events>
</Component>
<Component class="javax.swing.JLabel" name="searchstatus">
<Properties>
<Property name="horizontalAlignment" type="int" value="0"/>
<Property name="text" type="java.lang.String" value="Search Result"/>
</Properties>
</Component>
<Container class="javax.swing.JScrollPane" name="jScrollPane3">
<AuxValues>
<AuxValue name="autoScrollPane" type="java.lang.Boolean" value="true"/>
</AuxValues>
<Layout class="org.netbeans.modules.form.compat2.layouts.support.JScrollPaneSupportLayout"/>
<SubComponents>
<Component class="javax.swing.JTable" name="resultTable">
<Properties>
<Property name="model" type="javax.swing.table.TableModel" editor="org.netbeans.modules.form.editors2.TableModelEditor">
<Table columnCount="4" rowCount="0">
<Column editable="false" title="Tilte" type="java.lang.String"/>
<Column editable="false" title="Url" type="java.lang.String"/>
<Column editable="false" title="Description" type="java.lang.String"/>
<Column editable="false" title="Rating" type="java.lang.String"/>
</Table>
</Property>
<Property name="columnModel" type="javax.swing.table.TableColumnModel" editor="org.netbeans.modules.form.editors2.TableColumnModelEditor">
<TableColumnModel selectionModel="0">
<Column maxWidth="-1" minWidth="-1" prefWidth="200" resizable="true">
<Title/>
<Editor/>
<Renderer/>
</Column>
<Column maxWidth="-1" minWidth="-1" prefWidth="300" resizable="true">
<Title/>
<Editor/>
<Renderer/>
</Column>
<Column maxWidth="-1" minWidth="-1" prefWidth="500" resizable="true">
<Title/>
<Editor/>
<Renderer/>
</Column>
<Column maxWidth="-1" minWidth="-1" prefWidth="-1" resizable="true">
<Title/>
<Editor/>
<Renderer/>
</Column>
</TableColumnModel>
</Property>
<Property name="tableHeader" type="javax.swing.table.JTableHeader" editor="org.netbeans.modules.form.editors2.JTableHeaderEditor">
<TableHeader reorderingAllowed="true" resizingAllowed="true"/>
</Property>
</Properties>
</Component>
</SubComponents>
</Container>
<Container class="javax.swing.JScrollPane" name="jScrollPane1">
<AuxValues>
<AuxValue name="autoScrollPane" type="java.lang.Boolean" value="true"/>
</AuxValues>
<Layout class="org.netbeans.modules.form.compat2.layouts.support.JScrollPaneSupportLayout"/>
<SubComponents>
<Component class="javax.swing.JTable" name="languagetable">
<Properties>
<Property name="model" type="javax.swing.table.TableModel" editor="org.netbeans.modules.form.editors2.TableModelEditor">
<Table columnCount="1" rowCount="0">
<Column editable="false" title="Languages" type="java.lang.String"/>
</Table>
</Property>
<Property name="columnModel" type="javax.swing.table.TableColumnModel" editor="org.netbeans.modules.form.editors2.TableColumnModelEditor">
<TableColumnModel selectionModel="0">
<Column maxWidth="-1" minWidth="-1" prefWidth="-1" resizable="false">
<Title/>
<Editor/>
<Renderer/>
</Column>
</TableColumnModel>
</Property>
<Property name="tableHeader" type="javax.swing.table.JTableHeader" editor="org.netbeans.modules.form.editors2.JTableHeaderEditor">
<TableHeader reorderingAllowed="true" resizingAllowed="true"/>
</Property>
</Properties>
</Component>
</SubComponents>
</Container>
</SubComponents>
</Container>
</SubComponents>
</Form>
src/main/java/Main/Scraper.java 0 → 100644
/*
* To change this license header, choose License Headers in Project Properties.
* To change this template file, choose Tools | Templates
* and open the template in the editor.
*/
package Main;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.html.HtmlElement;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import java.net.URLEncoder;
import java.util.List;
import javax.swing.JOptionPane;
import javax.swing.table.DefaultTableModel;
/**
*
* @author HD Jay
*/
public class Scraper extends javax.swing.JFrame {
String baseUrl;
WebClient client;
/**
* Creates new form Scraper
*/
public Scraper() {
baseUrl = "https://github.com/";
client = new WebClient();
client.getOptions().setCssEnabled(false);
client.getOptions().setJavaScriptEnabled(false);
initComponents();
}
/**
* This method is called from within the constructor to initialize the form.
* WARNING: Do NOT modify this code. The content of this method is always
* regenerated by the Form Editor.
*/
@SuppressWarnings("unchecked")
// <editor-fold defaultstate="collapsed" desc="Generated Code">//GEN-BEGIN:initComponents
private void initComponents() {
jPanel1 = new javax.swing.JPanel();
jLabel1 = new javax.swing.JLabel();
searchText = new javax.swing.JTextField();
jButton1 = new javax.swing.JButton();
previous = new javax.swing.JButton();
pageNo = new javax.swing.JLabel();
next = new javax.swing.JButton();
clear = new javax.swing.JButton();
searchstatus = new javax.swing.JLabel();
jScrollPane3 = new javax.swing.JScrollPane();
resultTable = new javax.swing.JTable();
jScrollPane1 = new javax.swing.JScrollPane();
languagetable = new javax.swing.JTable();
setDefaultCloseOperation(javax.swing.WindowConstants.EXIT_ON_CLOSE);
jPanel1.setBackground(new java.awt.Color(153, 153, 153));
jLabel1.setFont(new java.awt.Font("Tahoma", 1, 14)); // NOI18N
jLabel1.setForeground(new java.awt.Color(255, 255, 255));
jLabel1.setText("WebScapper For GitHub By Hindu Singh");
searchText.setToolTipText("search");
jButton1.setText("Search");
jButton1.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
jButton1ActionPerformed(evt);
}
});
previous.setText("Previous");
previous.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
previousActionPerformed(evt);
}
});
pageNo.setText("1");
next.setText("Next");
next.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
nextActionPerformed(evt);
}
});
clear.setText("Clear");
clear.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
clearActionPerformed(evt);
}
});
searchstatus.setHorizontalAlignment(javax.swing.SwingConstants.CENTER);
searchstatus.setText("Search Result");
resultTable.setModel(new javax.swing.table.DefaultTableModel(
new Object [][] {
},
new String [] {
"Tilte", "Url", "Description", "Rating"
}
) {
Class[] types = new Class [] {
java.lang.String.class, java.lang.String.class, java.lang.String.class, java.lang.String.class
};
boolean[] canEdit = new boolean [] {
false, false, false, false
};
public Class getColumnClass(int columnIndex) {
return types [columnIndex];
}
public boolean isCellEditable(int rowIndex, int columnIndex) {
return canEdit [columnIndex];
}
});
jScrollPane3.setViewportView(resultTable);
if (resultTable.getColumnModel().getColumnCount() > 0) {
resultTable.getColumnModel().getColumn(0).setPreferredWidth(200);
resultTable.getColumnModel().getColumn(1).setPreferredWidth(300);
resultTable.getColumnModel().getColumn(2).setPreferredWidth(500);
}
languagetable.setModel(new javax.swing.table.DefaultTableModel(
new Object [][] {
},
new String [] {
"Languages"
}
) {
Class[] types = new Class [] {
java.lang.String.class
};
boolean[] canEdit = new boolean [] {
false
};
public Class getColumnClass(int columnIndex) {
return types [columnIndex];
}
public boolean isCellEditable(int rowIndex, int columnIndex) {
return canEdit [columnIndex];
}
});
jScrollPane1.setViewportView(languagetable);
if (languagetable.getColumnModel().getColumnCount() > 0) {
languagetable.getColumnModel().getColumn(0).setResizable(false);
}
javax.swing.GroupLayout jPanel1Layout = new javax.swing.GroupLayout(jPanel1);
jPanel1.setLayout(jPanel1Layout);
jPanel1Layout.setHorizontalGroup(
jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(jPanel1Layout.createSequentialGroup()
.addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(jPanel1Layout.createSequentialGroup()
.addGap(315, 315, 315)
.addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(jLabel1)
.addGroup(jPanel1Layout.createSequentialGroup()
.addComponent(searchText, javax.swing.GroupLayout.PREFERRED_SIZE, 155, javax.swing.GroupLayout.PREFERRED_SIZE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING, false)
.addComponent(clear, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
.addComponent(jButton1, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))))
.addGap(137, 137, 137))
.addGroup(javax.swing.GroupLayout.Alignment.TRAILING, jPanel1Layout.createSequentialGroup()
.addGap(20, 20, 20)
.addComponent(jScrollPane3)))
.addGroup(jPanel1Layout.createSequentialGroup()
.addGap(96, 96, 96)
.addComponent(searchstatus, javax.swing.GroupLayout.PREFERRED_SIZE, 485, javax.swing.GroupLayout.PREFERRED_SIZE)))
.addGap(15, 15, 15)
.addComponent(jScrollPane1, javax.swing.GroupLayout.DEFAULT_SIZE, 193, Short.MAX_VALUE)
.addContainerGap())
.addGroup(jPanel1Layout.createSequentialGroup()
.addGap(249, 249, 249)
.addComponent(previous)
.addGap(18, 18, 18)
.addComponent(pageNo, javax.swing.GroupLayout.PREFERRED_SIZE, 27, javax.swing.GroupLayout.PREFERRED_SIZE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addComponent(next, javax.swing.GroupLayout.PREFERRED_SIZE, 70, javax.swing.GroupLayout.PREFERRED_SIZE)
.addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE))
);
jPanel1Layout.setVerticalGroup(
jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addGroup(jPanel1Layout.createSequentialGroup()
.addContainerGap()
.addComponent(jLabel1)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
.addComponent(searchText, javax.swing.GroupLayout.PREFERRED_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.PREFERRED_SIZE)
.addComponent(jButton1))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addComponent(clear)
.addGap(36, 36, 36)
.addComponent(searchstatus, javax.swing.GroupLayout.PREFERRED_SIZE, 22, javax.swing.GroupLayout.PREFERRED_SIZE)
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED)
.addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(jScrollPane1, javax.swing.GroupLayout.DEFAULT_SIZE, 344, Short.MAX_VALUE)
.addComponent(jScrollPane3, javax.swing.GroupLayout.PREFERRED_SIZE, 0, Short.MAX_VALUE))
.addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED)
.addGroup(jPanel1Layout.createParallelGroup(javax.swing.GroupLayout.Alignment.BASELINE)
.addComponent(previous)
.addComponent(pageNo)
.addComponent(next))
.addContainerGap())
);
javax.swing.GroupLayout layout = new javax.swing.GroupLayout(getContentPane());
getContentPane().setLayout(layout);
layout.setHorizontalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(jPanel1, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
);
layout.setVerticalGroup(
layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING)
.addComponent(jPanel1, javax.swing.GroupLayout.DEFAULT_SIZE, javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)
);
pack();
}// </editor-fold>//GEN-END:initComponents
private void previousActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_previousActionPerformed
int pageno = Integer.parseInt(pageNo.getText());
if(pageno>1){
pageno--;
pageNo.setText(String.valueOf(pageno));
Search(pageno);
}else{
JOptionPane.showMessageDialog(this,"On first Page Already,","Can't search",JOptionPane.WARNING_MESSAGE);
}
}//GEN-LAST:event_previousActionPerformed
private void nextActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_nextActionPerformed
int pageno = Integer.parseInt(pageNo.getText());
DefaultTableModel result = (DefaultTableModel) resultTable.getModel();
if(result.getRowCount()==0){
JOptionPane.showMessageDialog(this,"Search first,","Can't search",JOptionPane.WARNING_MESSAGE);
}else{
pageno++;
pageNo.setText(String.valueOf(pageno));
Search(pageno);
}
}//GEN-LAST:event_nextActionPerformed
private void jButton1ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_jButton1ActionPerformed
int pageno = Integer.parseInt(pageNo.getText());
String q = searchText.getText();
if(q.isEmpty()){
JOptionPane.showMessageDialog(this,"Please enter query first,","Can't search",JOptionPane.WARNING_MESSAGE);
}else{
Search(pageno);
}
}//GEN-LAST:event_jButton1ActionPerformed
private void clearActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_clearActionPerformed
DefaultTableModel result = (DefaultTableModel) resultTable.getModel();
result.setRowCount(0);
DefaultTableModel lang = (DefaultTableModel) languagetable.getModel();
lang.setRowCount(0);
searchText.setText("");
pageNo.setText("1");
searchstatus.setText("Search Result");
}//GEN-LAST:event_clearActionPerformed
public void Search (int pageNo){
String searchQuery = searchText.getText();
DefaultTableModel result = (DefaultTableModel) resultTable.getModel();
DefaultTableModel language = (DefaultTableModel) languagetable.getModel();
result.setRowCount(0);
language.setRowCount(0);
try {
String searchUrl = baseUrl + "search?p="+pageNo+"?utf8=%E2%9C%93&q=" + URLEncoder.encode(searchQuery, "UTF-8");
HtmlPage page = client.getPage(searchUrl);
List<HtmlElement> header = (List<HtmlElement>) page.getByXPath("//div[@class='d-flex flex-justify-between border-bottom pb-3']");
HtmlElement repoNo = (HtmlElement) header.get(0).getFirstByXPath(".//h3");
List<HtmlElement> items = (List<HtmlElement>) page.getByXPath("//div[@class='repo-list-item d-flex flex-justify-start py-4 public source']") ;
if(items.isEmpty()){
System.out.println("No items found ! search with different query");
searchstatus.setText("No items found ! search with different query");
}else{
searchstatus.setText(repoNo.asText());
for(HtmlElement htmlItem : items){
HtmlAnchor itemAnchor = ((HtmlAnchor) htmlItem.getFirstByXPath(".//a[@class='v-align-middle']"));
HtmlElement itemDesc = ((HtmlElement) htmlItem.getFirstByXPath(".//p")) ;
HtmlElement starRating = ((HtmlElement) htmlItem.getFirstByXPath(".//a[@class='muted-link']")) ;
String rating = starRating == null?"0" : starRating.asText();
String itemUrl = "https://github.com"+itemAnchor.getHrefAttribute() ;
Item item = new Item();
item.setTitle(itemAnchor.asText());
item.setUrl(itemUrl);
item.setDesc( itemDesc.asText());
item.setRating(rating);
result.addRow(
new Object[]{
String.valueOf(item.getTitle()),
String.valueOf(item.getUrl()),
String.valueOf(item.getDesc()),
String.valueOf(item.getRating())
});
item.setRating(rating);
ObjectMapper mapper = new ObjectMapper();
String jsonString = mapper.writeValueAsString(item) ;
System.out.println(jsonString);
}
}
List<HtmlElement> languageList = (List<HtmlElement>) page.getByXPath("//ul[@class='filter-list small']");
List<HtmlElement> listLang = null;
if(languageList == null || languageList.isEmpty()){
System.out.println("No languages found !");System.out.println("No languages found !");
language.addRow(
new Object[]{
String.valueOf("No languages found!")
});
}else{
listLang = (List<HtmlElement>) languageList.get(0).getByXPath(".//li");
}
if(listLang == null || listLang.isEmpty()){
System.out.println("No languages found !");
}else{
for(HtmlElement htmlItem : listLang){
HtmlAnchor itemAnchor = ((HtmlAnchor) htmlItem.getFirstByXPath(".//a[@class='filter-item']"));
Language lang = new Language();
lang.setName((itemAnchor.asText()));
language.addRow(
new Object[]{
String.valueOf(lang.getName())
});
ObjectMapper mapper = new ObjectMapper();
String jsonString = mapper.writeValueAsString(lang) ;
System.out.println(jsonString);
}
}
} catch(Exception e){
e.printStackTrace();
}
}
/**
* @param args the command line arguments
*/
public static void main(String args[]) {
/* Set the Nimbus look and feel */
//<editor-fold defaultstate="collapsed" desc=" Look and feel setting code (optional) ">
/* If Nimbus (introduced in Java SE 6) is not available, stay with the default look and feel.
* For details see http://download.oracle.com/javase/tutorial/uiswing/lookandfeel/plaf.html
*/
try {
for (javax.swing.UIManager.LookAndFeelInfo info : javax.swing.UIManager.getInstalledLookAndFeels()) {
if ("Nimbus".equals(info.getName())) {
javax.swing.UIManager.setLookAndFeel(info.getClassName());
break;
}
}
} catch (ClassNotFoundException ex) {
java.util.logging.Logger.getLogger(Scraper.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
} catch (InstantiationException ex) {
java.util.logging.Logger.getLogger(Scraper.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
} catch (IllegalAccessException ex) {
java.util.logging.Logger.getLogger(Scraper.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
} catch (javax.swing.UnsupportedLookAndFeelException ex) {
java.util.logging.Logger.getLogger(Scraper.class.getName()).log(java.util.logging.Level.SEVERE, null, ex);
}
//</editor-fold>
/* Create and display the form */
java.awt.EventQueue.invokeLater(new Runnable() {
public void run() {
new Scraper().setVisible(true);
}
});
}
// Variables declaration - do not modify//GEN-BEGIN:variables
private javax.swing.JButton clear;
private javax.swing.JButton jButton1;
private javax.swing.JLabel jLabel1;
private javax.swing.JPanel jPanel1;
private javax.swing.JScrollPane jScrollPane1;
private javax.swing.JScrollPane jScrollPane3;
private javax.swing.JTable languagetable;
private javax.swing.JButton next;
private javax.swing.JLabel pageNo;
private javax.swing.JButton previous;
private javax.swing.JTable resultTable;
private javax.swing.JTextField searchText;
private javax.swing.JLabel searchstatus;
// End of variables declaration//GEN-END:variables
}
target/classes/Main/Item.class 0 → 100644
No preview for this file type
target/classes/Main/Language.class 0 → 100644
No preview for this file type
target/classes/Main/Scraper$1.class 0 → 100644
No preview for this file type
target/classes/Main/Scraper$2.class 0 → 100644
No preview for this file type
target/classes/Main/Scraper$3.class 0 → 100644
No preview for this file type
target/classes/Main/Scraper$4.class 0 → 100644
No preview for this file type
target/classes/Main/Scraper$5.class 0 → 100644
No preview for this file type
target/classes/Main/Scraper$6.class 0 → 100644
No preview for this file type
target/classes/Main/Scraper$7.class 0 → 100644
No preview for this file type
target/classes/Main/Scraper.class 0 → 100644
No preview for this file type
target/maven-archiver/pom.properties 0 → 100644
#Created by Apache Maven 3.0.5
version=0.0.1
groupId=hdjay
artifactId=webscraper
target/webscraper-0.0.1-jar-with-dependencies.jar 0 → 100644
The file could not be displayed because it is too large.
target/webscraper-0.0.1.jar 0 → 100644
No preview for this file type
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!