added gitdashboard project

README.md

@@ -4,8 +4,10 @@ GitHub Dashboard is Web Results Parser Web application which parse the GitHub se
 
 
 GitHub Dashboard identifies the following :
 GitHub Dashboard identifies the following :
 
 
-Github url and description of top 50 projects having highest stars. 
+Github url and description of top 50 projects having highest stars.
+ 
 Unique list of programming languages involved in top 50 projects.
 Unique list of programming languages involved in top 50 projects.
+
 Average star rating of all top 50 projects.
 Average star rating of all top 50 projects.
 
 
 ## Building & Running application 
 ## Building & Running application 

github-dashboard/pom.xml

+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<groupId>github-dashboard</groupId>
+	<artifactId>github-dashboard</artifactId>
+	<version>0.0.1-SNAPSHOT</version>
+
+	<parent>
+		<groupId>org.springframework.boot</groupId>
+		<artifactId>spring-boot-starter-parent</artifactId>
+		<version>1.5.7.RELEASE</version>
+	</parent>
+	<dependencies>
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter-web</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter-thymeleaf</artifactId>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.httpcomponents</groupId>
+			<artifactId>httpclient</artifactId>
+		</dependency>
+
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<scope>test</scope>
+		</dependency>
+
+		<dependency>
+			<groupId>us.codecraft</groupId>
+			<artifactId>xsoup</artifactId>
+			<version>0.3.1</version>
+		</dependency>
+
+
+	</dependencies>
+
+	<build>
+		<finalName>github-dashboard</finalName>
+		<plugins>
+			<plugin>
+				<groupId>org.springframework.boot</groupId>
+				<artifactId>spring-boot-maven-plugin</artifactId>
+			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-compiler-plugin</artifactId>
+				<version>3.7.0</version>
+				<configuration>
+					<source>1.8</source>
+					<target>1.8</target>
+				</configuration>
+			</plugin>
+		</plugins>
+	</build>
+
+	<repositories>
+		<repository>
+			<id>spring-releases</id>
+			<name>Spring Releases</name>
+			<url>https://repo.spring.io/libs-release</url>
+		</repository>
+		<repository>
+			<id>org.jboss.repository.releases</id>
+			<name>JBoss Maven Release Repository</name>
+			<url>https://repository.jboss.org/nexus/content/repositories/releases</url>
+		</repository>
+	</repositories>
+
+	<pluginRepositories>
+		<pluginRepository>
+			<id>spring-releases</id>
+			<name>Spring Releases</name>
+			<url>https://repo.spring.io/libs-release</url>
+		</pluginRepository>
+	</pluginRepositories>
+
+</project>
\ No newline at end of file

github-dashboard/src/main/java/com/github/dashboard/app/App.java

+package com.github.dashboard.app;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+@SpringBootApplication
+public class App {
+	
+	
+	public static void main(String args[]) {
+		
+		SpringApplication.run(App.class, args);
+
+	}
+
+}

github-dashboard/src/main/java/com/github/dashboard/app/config/Appconfig.java

+package com.github.dashboard.app.config;
+
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.PropertySource;
+
+@Configuration
+@PropertySource("application.properties")
+public class Appconfig {
+
+	
+}

github-dashboard/src/main/java/com/github/dashboard/app/controller/DashboardController.java

+package com.github.dashboard.app.controller;
+
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.RequestMapping;
+
+@Controller
+public class DashboardController {
+	
+	@RequestMapping("/dashboard")
+	public String home() {
+		return "dashboard";
+	}
+
+}

github-dashboard/src/main/java/com/github/dashboard/app/controller/SearchController.java

+package com.github.dashboard.app.controller;
+
+import java.util.List;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+import com.github.dashboard.app.model.Repository;
+import com.github.dashboard.app.model.SearchRequest;
+import com.github.dashboard.app.model.SearchResponse;
+import com.github.dashboard.app.service.SearchService;
+
+@RestController
+public class SearchController {
+
+	@Autowired
+	SearchService searchService;
+
+	@RequestMapping(value = "/search", method = RequestMethod.GET)
+	public SearchResponse search(@RequestParam String query) {
+
+		SearchResponse searchResponse = new SearchResponse();
+		
+		SearchRequest searchRequest = new SearchRequest();
+		searchRequest.setQuery(query);
+		searchRequest.setSort("stars");
+		
+		List<Repository>  repositories = searchService.search(searchRequest);
+		
+		searchResponse.setRepositories(repositories);
+		searchResponse.setLanguages(searchService.getUniqueLanguages(repositories));
+		searchResponse.setAvgStars(searchService.getAverageStars(repositories));
+		
+		return searchResponse;
+	}
+
+}

github-dashboard/src/main/java/com/github/dashboard/app/model/Repository.java

+package com.github.dashboard.app.model;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class Repository {
+
+	private String url;
+	private String description;
+	private String language;
+	private float star;
+
+	public String getUrl() {
+		return url;
+	}
+
+	public void setUrl(String url) {
+		this.url = url;
+	}
+
+	public String getDescription() {
+		return description;
+	}
+
+	public void setDescription(String description) {
+		this.description = description;
+	}
+		
+	public String getLanguage() {
+		return language;
+	}
+
+	public void setLanguage(String language) {
+		this.language = language;
+	}
+	
+	public float getStar() {
+		return star;
+	}
+
+	public void setStar(float star) {
+		this.star = star;
+	}
+
+	@Override
+	public String toString() {
+		return "Repository [url=" + url + ", description=" + description + ", language=" + language + ", star=" + star
+				+ "]";
+	}
+
+	
+
+}

github-dashboard/src/main/java/com/github/dashboard/app/model/SearchRequest.java

+package com.github.dashboard.app.model;
+
+public class SearchRequest {
+
+	private String query;
+	private String sort;
+
+	public String getQuery() {
+		return query;
+	}
+
+	public void setQuery(String query) {
+		this.query = query;
+	}
+
+	public String getSort() {
+		return sort;
+	}
+
+	public void setSort(String sort) {
+		this.sort = sort;
+	}
+
+
+}

github-dashboard/src/main/java/com/github/dashboard/app/model/SearchResponse.java

+package com.github.dashboard.app.model;
+
+import java.util.List;
+import java.util.Set;
+
+public class SearchResponse {
+
+	private List<Repository> repositories;
+	private Set<String> languages;
+	private double avgStars;
+
+	public List<Repository> getRepositories() {
+		return repositories;
+	}
+
+	public void setRepositories(List<Repository> repositories) {
+		this.repositories = repositories;
+	}
+
+	public Set<String> getLanguages() {
+		return languages;
+	}
+
+	public void setLanguages(Set<String> languages) {
+		this.languages = languages;
+	}
+
+	public double getAvgStars() {
+		return avgStars;
+	}
+
+	public void setAvgStars(double avgStars) {
+		this.avgStars = avgStars;
+	}
+
+	@Override
+	public String toString() {
+		return "SearchResponse [repositories=" + repositories + ", languages=" + languages + ", avgStars=" + avgStars
+				+ "]";
+	}
+	
+	
+
+}

github-dashboard/src/main/java/com/github/dashboard/app/service/SearchService.java

+package com.github.dashboard.app.service;
+
+import java.util.List;
+import java.util.Set;
+
+import com.github.dashboard.app.model.Repository;
+import com.github.dashboard.app.model.SearchRequest;
+
+public interface SearchService {
+
+	public List<Repository> search(SearchRequest searchRequest);
+
+	public Set<String> getUniqueLanguages(List<Repository> repositories);
+
+	public double getAverageStars(List<Repository> repositories);
+
+}

github-dashboard/src/main/java/com/github/dashboard/app/serviceimpl/SearchServiceImpl.java

+package com.github.dashboard.app.serviceimpl;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+import org.springframework.util.StringUtils;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import com.github.dashboard.app.model.Repository;
+import com.github.dashboard.app.model.SearchRequest;
+import com.github.dashboard.app.service.SearchService;
+import com.github.dashboard.app.util.HTMLUtil;
+
+@Service
+public class SearchServiceImpl implements SearchService {
+
+	
+	@Value("${url}")
+	private String url;
+
+	@Override
+	public List<Repository> search(SearchRequest searchRequest) {
+
+		String pageUrl;
+		List<Repository> repositories = new ArrayList<>();
+
+		for (int index = 1; index <= 5; index++) {
+
+			pageUrl = getPageUrl(searchRequest, index);
+			repositories.addAll(HTMLUtil.extractRepositories(pageUrl));
+		}
+
+		return repositories;
+	}
+
+	private String getPageUrl(SearchRequest searchRequest, int pageNo) {
+
+		UriComponentsBuilder builder = UriComponentsBuilder.fromUriString(url)
+				// Add query parameter
+				.queryParam("q", searchRequest.getQuery() != null ? searchRequest.getQuery() : "")
+				.queryParam("s", searchRequest.getSort()).queryParam("p", pageNo);
+
+		return builder.build().toUri().toString();
+	}
+
+	@Override
+	public Set<String> getUniqueLanguages(List<Repository> repositories) {
+
+		Set<String> languages = new HashSet<>();
+		repositories.forEach(repository -> {
+
+			if (StringUtils.hasText(repository.getLanguage())) {
+				languages.add(repository.getLanguage());
+			}
+		});
+
+		return languages;
+	}
+
+	@Override
+	public double getAverageStars(List<Repository> repositories) {
+
+		return repositories.stream()
+				.mapToDouble(Repository::getStar)
+				.average().getAsDouble();
+
+	}
+
+}

github-dashboard/src/main/java/com/github/dashboard/app/util/HTMLUtil.java

+package com.github.dashboard.app.util;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import org.springframework.util.CollectionUtils;
+import org.springframework.util.StringUtils;
+
+import com.github.dashboard.app.model.Repository;
+
+public class HTMLUtil {
+
+	public static List<Repository> extractRepositories(String path) {
+
+		List<Repository> repositories = new ArrayList<>();	
+
+		try {
+
+			Document document = Jsoup.connect(path).get();
+
+			Elements reportList = document.select(".repo-list-item");
+
+			reportList.forEach(reportListItem -> {
+
+				Repository repository = new Repository();
+
+				addRepositoryUrl(reportListItem, repository);
+				addRepositoryDescription(reportListItem, repository);
+				addRepositoryLanguage(reportListItem, repository);
+				addRepositoryStar(reportListItem, repository);
+
+				repositories.add(repository);
+			});
+
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+
+		return repositories;
+	}
+
+	private static void addRepositoryUrl(Element reportListItem, Repository repository) {
+
+		Elements url = reportListItem.select("div > h3 > a");
+
+		if (!CollectionUtils.isEmpty(url)) {
+
+			repository.setUrl("https://github.com" + url.get(0).attr("href"));
+		}
+		
+		if(!StringUtils.hasText(repository.getUrl())) {
+			repository.setDescription("No url present");
+		}
+	}
+
+	private static void addRepositoryDescription(Element reportListItem, Repository repository) {
+
+		Elements description = reportListItem.select("div > p.col-9");
+
+		if (!CollectionUtils.isEmpty(description)) {
+
+			repository.setDescription(description.get(0).text());
+		}
+		
+		if(!StringUtils.hasText(repository.getDescription())) {
+			repository.setDescription("No description provided");
+		}
+		
+
+	}
+
+	private static void addRepositoryLanguage(Element reportListItem, Repository repository) {
+
+		Elements language = reportListItem.select("div.d-table-cell");
+
+		if (!CollectionUtils.isEmpty(language)) {
+
+			repository.setLanguage(language.get(0).text());
+		}
+	}
+
+	private static void addRepositoryStar(Element reportListItem, Repository repository) {
+		
+		Elements star = reportListItem.select("div > a.muted-link");
+
+		if (!CollectionUtils.isEmpty(star)) {
+
+			repository.setStar(getStarValueFromText(star.get(0).text()));
+		}
+
+	}
+
+	private static float getStarValueFromText(String text) {
+
+		float value = 0f;
+		int multFactor = 1;
+
+		if (StringUtils.hasText(text)) {
+
+			if (text.trim().toUpperCase().endsWith("K")) {
+				multFactor = 1000;
+			} else if (text.trim().toUpperCase().endsWith("M")) {
+				multFactor = 1000 * 1000;
+			}
+
+			value = Float.parseFloat(text.trim().replaceAll("[^0-9]+", "")) * multFactor;
+		}
+
+		return value;
+	}
+}

github-dashboard/src/main/resources/application.properties

+url=https://github.com/search
+
+
+
+
+#https://github.com/search?q=artifical+intelligence&type=Repositories&p=1

github-dashboard/src/main/resources/static/app.js

+
+
+function load(query) {
+	
+	$("#loading").show();
+	
+	$.get("/search?query=" + query , function(data){
+		
+		$('#repo-table').DataTable({
+	        data: data.repositories,
+	        order: [[ 2, "desc" ]],
+	        destroy: true,
+	        columns: [
+	            { title: "Url", data : "url" , "render": function ( data, type, row ) {
+                    return "<a href='"+ data + "'>" + data + "</a>";
+                }},
+	            { title: "Description" , data : "description"  },
+	            { title: "stars" , data : "star"  , "visible": false }
+	        ]
+	    });
+		
+		
+		$('#lang').html("");
+		
+		$.each(data.languages, function (index, item) {
+		    $('#lang').append(item);
+		    if(index < data.languages.length - 1) {
+		    	$('#lang').append(",");
+		    } else {
+		    	$('#lang').append(".");
+		    }
+		});
+		
+		$('#avg-star').html(data.avgStars);
+		
+		$("#loading").hide();
+	});
+	
+}
+
+$(document).ready(function(){
+	
+	$("#search-box").val("artifical intelligence")
+	load("artifical intelligence");
+	
+	$("#search-button").click(function(){
+		
+		$("#loading").show();
+		
+		query = $("#search-box").val();
+		
+		if(query.replace(/\s/g, "") !== ''){
+			load(query);
+		}
+		
+		
+	});
+	
+	$('#search-box').keydown(function(event){ 
+	    var keyCode = (event.keyCode ? event.keyCode : event.which);   
+	    if (keyCode == 13) {
+	        $('#search-button').trigger('click');
+	    }
+	});
+	
+});
+
+
+

github-dashboard/src/main/resources/templates/dashboard.html

+<!DOCTYPE HTML>
+<html xmlns:th="http://www.thymeleaf.org">
+<head>
+<title>GitHub DashBoard</title>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+
+
+
+
+
+
+<link rel="stylesheet"
+	href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/4.0.0-beta/css/bootstrap.css">
+</link>
+
+<link rel="stylesheet"
+	href="https://cdn.datatables.net/1.10.16/css/jquery.dataTables.min.css">
+</link>
+
+<script src="https://code.jquery.com/jquery-1.12.4.js"></script>
+
+<script
+	src="https://cdn.datatables.net/1.10.16/js/jquery.dataTables.min.js"></script>
+
+<script
+	src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-beta/js/bootstrap.min.js"
+	integrity="sha384-h0AbiXch4ZDo7tp9hKZ4TsHbi047NrKGLO3SEJAg45jXxnGIfYzk4Si90RDIqNm1"
+	crossorigin="anonymous"></script>
+
+
+
+<script
+	src="https://cdn.datatables.net/1.10.16/js/dataTables.bootstrap4.min.js"></script>
+
+
+
+<script src="/app.js"></script>
+
+
+</head>
+<body>
+
+	<div class="container">
+
+		<ol class="breadcrumb" style="background: #888">
+			<li class="breadcrumb-item active" style="color: white">GitHub Dashboard</li>
+		</ol>
+
+
+
+		<div class="content">
+
+			<div class="row">
+				<div class="col-lg-6">
+					<div class="input-group">
+						<input type="text" class="form-control"
+							placeholder="Search for..." id="search-box"> </input><span
+							class="input-group-btn">
+							<button class="btn btn-secondary" type="button" id="search-button">Search</button>
+							
+						</span>
+					</div>
+					<span id="loading" style="color: green;">Loading...</span>
+				</div>
+			</div>
+
+			<div style="padding-top: 3%;"></div>
+			<div class="alert alert-success" role="alert">
+				<div>Languages in Top 50 : <span id="lang"></span></div> <div>Average Star Rating : <span id="avg-star"></span></div>
+			</div>
+
+			<div id="repo-table-div" style="padding-top: 3%;">
+				<table id="repo-table" class="cell-border">
+				</table>
+			</div>
+
+			<span style="width: 50%;"> </span>
+
+
+		</div>
+
+	</div>
+	
+	
+
+</body>
+</html>
\ No newline at end of file