SearchSECODatabaseAPI
JobRequestHandler.h
1/*
2This program has been developed by students from the bachelor Computer Science at
3Utrecht University within the Software Project course.
4© Copyright Utrecht University (Department of Information and Computing Sciences)
5*/
6
7#pragma once
8#include "DatabaseConnection.h"
9#include "RAFTConsensus.h"
10
11#include <mutex>
12#include <boost/shared_ptr.hpp>
13
14#define MIN_AMOUNT_JOBS 500
15#define MAX_RETRIES 3
16#define CRAWL_TIMEOUT_SECONDS 150
17#define NO_RETRY_REASONS {10}
18
19class TcpConnection;
20
22{
23public:
27 JobRequestHandler(RAFTConsensus *raft, RequestHandler *requestHandler, DatabaseConnection *database, Statistics *stats,
28 std::string ip, int port);
29
33 std::string handleConnectRequest(boost::shared_ptr<TcpConnection> connection, std::string request);
34
38 std::string handleGetIPsRequest(std::string request, std::string client, std::string data);
39
51 std::string handleUploadJobRequest(std::string request, std::string client, std::string data);
52
64 std::string handleUploadJobRequest(std::string request, std::string client, std::vector<std::string> data);
65
76 std::string handleGetJobRequest(std::string request, std::string client, std::string data);
77
84 std::string handleUpdateJobRequest(std::string request, std::string client, std::string data);
85
97 std::string handleFinishJobRequest(std::string request, std::string client, std::string data);
98
109 std::string handleCrawlDataRequest(std::string request, std::string client, std::string data);
110
117 long long timeLastCrawl = -1;
118
125 void updateCrawlID(int id);
126
127 DatabaseConnection *getDatabaseConnection()
128 {
129 return database;
130 }
131
132private:
133 RAFTConsensus *raft;
134 RequestHandler *requestHandler;
135 DatabaseConnection *database;
136 Statistics *stats;
137 std::mutex jobmtx;
138
144 void connectWithRetry(std::string ip, int port);
145
151 Job getTopJobWithRetry();
152
158 void tryUploadJobWithRetry(Job job, bool newMethod);
159
167 long long getCurrentJobTimeWithRetry(std::string jobid);
168
177 Job getCurrentJobWithRetry(std::string jobid);
178
182 long long addCurrentJobWithRetry(Job job);
183
187 void addFailedJobWithRetry(FailedJob job);
188};
Handles interaction with database when dealing with job requests.
Definition: DatabaseConnection.h:25
Definition: JobRequestHandler.h:22
std::string handleConnectRequest(boost::shared_ptr< TcpConnection > connection, std::string request)
Handles request from new node to connect to the network.
Definition: JobRequestHandler.cpp:29
std::string handleGetIPsRequest(std::string request, std::string client, std::string data)
Handles request for the ip adresses in the network.
Definition: JobRequestHandler.cpp:34
std::string handleUploadJobRequest(std::string request, std::string client, std::string data)
Handles request to upload one or more jobs with their priorities.
Definition: JobRequestHandler.cpp:238
JobRequestHandler(RAFTConsensus *raft, RequestHandler *requestHandler, DatabaseConnection *database, Statistics *stats, std::string ip, int port)
Constructor method.
Definition: JobRequestHandler.cpp:17
void updateCrawlID(int id)
Updates the crawlID when crawl data is uploaded to the database.
Definition: JobRequestHandler.cpp:359
std::string handleUpdateJobRequest(std::string request, std::string client, std::string data)
Handles the request to update the job time.
Definition: JobRequestHandler.cpp:102
std::string handleGetJobRequest(std::string request, std::string client, std::string data)
Handles request to give the top job from the queue.
Definition: JobRequestHandler.cpp:51
int crawlID
Variables describing the current crawlID which is needed by the crawler to crawl a specific part of G...
Definition: JobRequestHandler.h:116
std::string handleFinishJobRequest(std::string request, std::string client, std::string data)
Handles request to indicate the worker is finished with a job, successfully or not.
Definition: JobRequestHandler.cpp:153
std::string handleCrawlDataRequest(std::string request, std::string client, std::string data)
Handles request to upload crawl data to the job queue.
Definition: JobRequestHandler.cpp:308
Definition: RAFTConsensus.h:41
Definition: RequestHandler.h:43
Stores the Prometheus statistics variables.
Definition: Statistics.h:35
Definition: ConnectionHandler.h:56
Mirrors an entry of the failedjobs table in the database. Difference with Job is the addition of a re...
Definition: JobTypes.h:41
Mirrors an entry of the currentjobs table in the database.
Definition: JobTypes.h:16