Skip to content
This repository was archived by the owner on Oct 22, 2020. It is now read-only.

Commit d9a360c

Browse files
fakturkfakturk
committed
parsed pdf added to search result
1 parent 6e47e22 commit d9a360c

3 files changed

Lines changed: 33 additions & 95 deletions

File tree

‎src/main/java/io/academic/service/AcademicSearchService.java‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import com.google.gson.JsonParser;
77
import org.elasticsearch.action.search.*;
88
import org.elasticsearch.common.unit.TimeValue;
9+
import org.elasticsearch.index.query.QueryBuilder;
910
import org.elasticsearch.index.query.QueryBuilders;
1011
import org.elasticsearch.search.Scroll;
1112
import org.elasticsearch.search.SearchHit;
@@ -16,6 +17,8 @@
1617
import org.springframework.stereotype.Service;
1718

1819
import java.io.IOException;
20+
import java.util.ArrayList;
21+
import java.util.List;
1922

2023
import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
2124

@@ -30,8 +33,11 @@ public AcademicSearchService() {
3033

3134
public String search(String q) throws IOException {
3235

36+
ArrayList<String> criterias = new ArrayList<String>();
37+
criterias.add("dc");
38+
criterias.add("content");
3339
SearchRequest searchRequest = new SearchRequest("harvester");
34-
searchRequest.source(buildSource("term","dc",q,false));
40+
searchRequest.source(buildSource("term",criterias,q,false));
3541

3642
//this values are necessary if we need scrollable results (in other words if our result have more than 10 hits)
3743
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1));
@@ -47,8 +53,10 @@ public String search(String q) throws IOException {
4753

4854
public String searchBy(String q, String criteria) throws IOException {
4955

56+
ArrayList<String> criterias = new ArrayList<String>();
57+
criterias.add(criteria);
5058
SearchRequest searchRequest = new SearchRequest("harvester");
51-
searchRequest.source(buildSource("match",criteria,q,false));
59+
searchRequest.source(buildSource("match",criterias,q,false));
5260

5361
//this values are necessary if we need scrollable results (in other words if our result have more than 10 hits)
5462
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1));
@@ -66,8 +74,10 @@ public String searchBy(String q, String criteria) throws IOException {
6674

6775
public String getAll() throws IOException {
6876

77+
ArrayList<String> criterias = new ArrayList<String>();
78+
criterias.add("");
6979
SearchRequest searchRequest = new SearchRequest("harvester");
70-
searchRequest.source(buildSource("matchAll","","",true));
80+
searchRequest.source(buildSource("matchAll",criterias,"",true));
7181

7282
//this values are necessary if we need scrollable results (in other words if our result have more than 10 hits)
7383
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1));
@@ -128,16 +138,16 @@ public String toJson(String nonJsonString){
128138

129139

130140

131-
public SearchSourceBuilder buildSource(String queryType, String criteria, String q, Boolean showAllFields){
141+
public SearchSourceBuilder buildSource(String queryType, ArrayList<String> criteria, String q, Boolean showAllFields){
132142
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
133143

134144
if (queryType.equals("match"))
135145
{
136-
searchSourceBuilder.query(matchQuery(criteria,q));
146+
searchSourceBuilder.query(QueryBuilders.matchQuery(criteria.get(0),q));
137147
}
138148
else if (queryType.equals("term"))
139149
{
140-
searchSourceBuilder.query(QueryBuilders.termQuery(criteria,q));
150+
searchSourceBuilder.query(QueryBuilders.multiMatchQuery(q,criteria.toArray(new String[criteria.size()])));
141151
}
142152
else
143153
{
@@ -147,7 +157,8 @@ else if (queryType.equals("term"))
147157
searchSourceBuilder.sort(new FieldSortBuilder("title.keyword").order(SortOrder.DESC));
148158
if (!showAllFields)
149159
{
150-
String[] includeFields = new String[] {"title",criteria};
160+
criteria.add("title");
161+
String[] includeFields = criteria.toArray(new String[criteria.size()]);
151162
String[] excludeFields = new String[] {""};
152163
searchSourceBuilder.fetchSource(includeFields,excludeFields);
153164
searchSourceBuilder.fetchSource(true);

‎src/main/java/io/academic/service/OaiService.java‎

Lines changed: 14 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -90,67 +90,30 @@ public RestHighLevelClient getRestClient() {
9090
public static final String INDEX = "harvester";
9191
private static final String TYPE = "oai";
9292

93-
// private DocumentService documentService = null;
94-
// private IndexService indexService = null;
95-
// private SearchService searchService = null;
96-
// private IndexRequest request;
97-
98-
// @Autowired
99-
// public OaiService(DocumentService documentService, IndexService indexService, SearchService searchService) {
100-
// this.documentService = documentService;
101-
// this.indexService = indexService;
102-
//// indexService.createIndex();
103-
// this.searchService = searchService;
104-
// }
93+
10594
@Autowired
10695
public OaiService()
10796
{
10897

10998
}
11099

111100
public void elasticSave(Article article) throws IOException {
112-
// System.out.println("inside elasticsave");
113-
114-
// IndexRequest request = new IndexRequest(INDEX, TYPE).setEntity(article);
115-
// System.out.println("before article get Article Identifier");
116-
// System.out.println(article.getArticleIdentifier());
117-
// if (article.getArticleIdentifier() != null) {
118-
// request.setId(String.valueOf(article.getId()));
119-
// System.out.println("inside article getid");
120101
IndexRequest request = new IndexRequest(INDEX,TYPE);
121102
request.setPipeline("academic-pdf");
122-
// System.out.println(new Gson().toJson(article));
123-
request.source(new Gson().toJson(article), XContentType.JSON);
103+
// before using this pipeline we have to add pipeline to the elasticsearch by following command
104+
// PUT _ingest/pipeline/academic-pdf
105+
// {
106+
// "description": "parse pdfs and index into ES",
107+
// "processors" :
108+
// [
109+
// { "attachment" : { "field": "pdf" } },
110+
// { "remove" : { "field": "pdf" } }
111+
// ]
124112
// }
125113

126-
127-
114+
request.source(new Gson().toJson(article), XContentType.JSON);
128115
IndexResponse indexResponse = restClient.index(request);
129116

130-
// String index = indexResponse.getIndex();
131-
// String type = indexResponse.getType();
132-
// String id = indexResponse.getId();
133-
// long version = indexResponse.getVersion();
134-
// System.out.println(index+" ,"+type+", "+id+", "+version);
135-
// if (indexResponse.getResult() == DocWriteResponse.Result.CREATED) {
136-
//
137-
// } else if (indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
138-
//
139-
// }
140-
// ReplicationResponse.ShardInfo shardInfo = indexResponse.getShardInfo();
141-
// if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
142-
//
143-
// }
144-
// if (shardInfo.getFailed() > 0) {
145-
// for (ReplicationResponse.ShardInfo.Failure failure : shardInfo.getFailures()) {
146-
// String reason = failure.reason();
147-
// }
148-
// }
149-
150-
151-
152-
153-
// return documentService.index(request);
154117
}
155118

156119

@@ -179,11 +142,9 @@ public void saveRecords(List<RecordType> recordTypes) {
179142
oaiRecord.setState(0);
180143
oaiRecords.add(oaiRecord);
181144

145+
//TODO: we ave to check all the parts name and assigned according to related name not order
182146
String[] parts = parsedDc.getDc().split(";;");
183147
Article article = new Article();
184-
// System.out.println("article create sonrasi article id : "+article.getId());
185-
// System.out.println("article create sonrasi oai id : "+oaiRecord.getId());
186-
187148
article.setTitle(parts[0].split("::")[1]);
188149
article.setAuthors(parts[1].split("::")[1]);
189150
article.setKeywords(parts[2].split("::")[1]);
@@ -195,23 +156,18 @@ public void saveRecords(List<RecordType> recordTypes) {
195156
{
196157
String downlaodUrl = parts[10].split("::")[1];
197158
article.setRelation(downlaodUrl);
198-
// article.setBase64("not available");
199159
article.setBase64(UrlPdftoBase64(downlaodUrl));
200160
}
201161
else
202162
{
203163
article.setRelation("not available");
204-
article.setBase64("bm90IGF2YWlsYWJsZQ==");
164+
article.setBase64("bm90IGF2YWlsYWJsZQ=="); //it means not available in base 64
205165
}
206166
article.setDc(parsedDc.getDc());
207167
article.setArticleIdentifier(parseIdentifier(oaiRecord.getIdentifier()));
208-
// article.setArticleIdentifier(oaiRecord.getIdentifier());
209168

210-
// System.out.println("article add oncesi article id : "+article.getId());
211169
articles.add(article);
212170

213-
// System.out.println("elastic save oncesi article id : "+article.getId());
214-
// System.out.println("elastic save oncesi article title : "+article.getTitle());
215171
try {
216172
elasticSave(article);
217173
} catch (IOException e) {
@@ -247,13 +203,8 @@ public String UrlPdftoBase64(String url) {
247203
try {
248204
oracle = new URL(url);
249205
URLConnection yc = oracle.openConnection();
250-
// BufferedReader in = new BufferedReader(new InputStreamReader(
251-
// yc.getInputStream()));
206+
252207
BufferedInputStream bis = new BufferedInputStream(yc.getInputStream());
253-
// String inputLine;
254-
// while ((inputLine = in.readLine()) != null)
255-
// System.out.println(inputLine);
256-
// in.close();
257208

258209
byte bytes[] = IOUtils.toByteArray(bis);
259210
bis.close();
@@ -266,31 +217,8 @@ public String UrlPdftoBase64(String url) {
266217
e.printStackTrace();
267218
}
268219

269-
270-
271-
272220
return base64;
273221

274-
// String inputLine;
275-
// while ((inputLine = in.readLine()) != null)
276-
// System.out.println(inputLine);
277-
// in.close();
278-
279-
280-
281-
282-
// BufferedReader in = new BufferedReader(
283-
// new InputStreamReader(oracle.openStream()));
284-
// byte bytes[] = IOUtils.toByteArray(oracle);
285-
286-
287-
// String b64String = Base64.
288-
289-
// String inputLine;
290-
// while ((inputLine = in.readLine()) != null)
291-
// System.out.println(inputLine);
292-
// in.close();
293-
294222
}
295223

296224
private LocalDateTime parseDateTime(String string) {

‎src/main/java/io/academic/service/ProcessorService.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,7 @@ public void startProcessors() {
3838
oaiDataProviderService.queue(new OaiDataProvider("Acta Medica Anatolia","http://dergipark.gov.tr/api/public/oai/","http://dergipark.gov.tr/download/article-file/","dergipark.ulakbim.gov.tr" ));
3939
// oaiDataProviderService.queue(new OaiDataProvider("http://export.arxiv.org/oai2","https://arxiv.org/pdf/"));
4040

41-
oaiService.delete();
42-
41+
oaiService.delete();
4342

4443
}
4544

0 commit comments

Comments
 (0)