8#include "DatabaseConnection.h"
9#include "RAFTConsensus.h"
12#include <boost/shared_ptr.hpp>
14#define MIN_AMOUNT_JOBS 500
16#define CRAWL_TIMEOUT_SECONDS 150
17#define NO_RETRY_REASONS {10}
28 std::string ip,
int port);
33 std::string
handleConnectRequest(boost::shared_ptr<TcpConnection> connection, std::string request);
64 std::string
handleUploadJobRequest(std::string request, std::string client, std::vector<std::string> data);
117 long long timeLastCrawl = -1;
144 void connectWithRetry(std::string ip,
int port);
151 Job getTopJobWithRetry();
158 void tryUploadJobWithRetry(
Job job,
bool newMethod);
167 long long getCurrentJobTimeWithRetry(std::string jobid);
177 Job getCurrentJobWithRetry(std::string jobid);
182 long long addCurrentJobWithRetry(
Job job);
187 void addFailedJobWithRetry(
FailedJob job);
Handles interaction with database when dealing with job requests.
Definition: DatabaseConnection.h:25
Definition: JobRequestHandler.h:22
std::string handleConnectRequest(boost::shared_ptr< TcpConnection > connection, std::string request)
Handles request from new node to connect to the network.
Definition: JobRequestHandler.cpp:29
std::string handleGetIPsRequest(std::string request, std::string client, std::string data)
Handles request for the ip adresses in the network.
Definition: JobRequestHandler.cpp:34
std::string handleUploadJobRequest(std::string request, std::string client, std::string data)
Handles request to upload one or more jobs with their priorities.
Definition: JobRequestHandler.cpp:238
JobRequestHandler(RAFTConsensus *raft, RequestHandler *requestHandler, DatabaseConnection *database, Statistics *stats, std::string ip, int port)
Constructor method.
Definition: JobRequestHandler.cpp:17
void updateCrawlID(int id)
Updates the crawlID when crawl data is uploaded to the database.
Definition: JobRequestHandler.cpp:359
std::string handleUpdateJobRequest(std::string request, std::string client, std::string data)
Handles the request to update the job time.
Definition: JobRequestHandler.cpp:102
std::string handleGetJobRequest(std::string request, std::string client, std::string data)
Handles request to give the top job from the queue.
Definition: JobRequestHandler.cpp:51
int crawlID
Variables describing the current crawlID which is needed by the crawler to crawl a specific part of G...
Definition: JobRequestHandler.h:116
std::string handleFinishJobRequest(std::string request, std::string client, std::string data)
Handles request to indicate the worker is finished with a job, successfully or not.
Definition: JobRequestHandler.cpp:153
std::string handleCrawlDataRequest(std::string request, std::string client, std::string data)
Handles request to upload crawl data to the job queue.
Definition: JobRequestHandler.cpp:308
Definition: RAFTConsensus.h:41
Definition: RequestHandler.h:43
Stores the Prometheus statistics variables.
Definition: Statistics.h:35
Definition: ConnectionHandler.h:56
Mirrors an entry of the failedjobs table in the database. Difference with Job is the addition of a re...
Definition: JobTypes.h:41
Mirrors an entry of the currentjobs table in the database.
Definition: JobTypes.h:16