Scheduler

先看看文档对于Scheduler的作用介绍
https://code4craft.gitbooks.io/webmagic-in-action/content/zh/posts/ch1-overview/architecture.html
之前我们也介绍过了，Scheduler主要负责爬虫的下一步爬取的规划，包括一些去重等功能。在主流程中也看到了Scheduler，现在来具体结合源码分析

源码

Scheduler是一个接口

public interface Scheduler {

/**
* add a url to fetch
*
* @param request
* @param task
*/
public void push(Request request, Task task);

/**
* get an url to crawl
*
* @param task the task of spider
* @return the url to crawl
*/
public Request poll(Task task);

}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
其主要的实现是DuplicateRemovedScheduler，使用模板模式定义了push的步骤。

public abstract class DuplicateRemovedScheduler implements Scheduler {

protected Logger logger = LoggerFactory.getLogger(getClass());

private DuplicateRemover duplicatedRemover = new HashSetDuplicateRemover();

public DuplicateRemover getDuplicateRemover() {
return duplicatedRemover;
}

public DuplicateRemovedScheduler setDuplicateRemover(DuplicateRemover duplicatedRemover) {
this.duplicatedRemover = duplicatedRemover;
return this;
}

@Override
public void push(Request request, Task task) {
logger.trace("get a candidate url {}", request.getUrl());
if (!duplicatedRemover.isDuplicate(request, task) || shouldReserved(request)) {
logger.debug("push to queue {}", request.getUrl());
pushWhenNoDuplicate(request, task);
}
}

protected boolean shouldReserved(Request request) {
return request.getExtra(Request.CYCLE_TRIED_TIMES) != null;
}

protected void pushWhenNoDuplicate(Request request, Task task) {

}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
我们来看看负责去重的接口DuplicateRemover，其实现类有HashSetDuplicateRemover使用HashSet来去重，RedisScheduler接触Redis来去重和BloomFilterDuplicateRemover使用BloomFilter去重。默认使用HashSetDuplicateRemover

public class HashSetDuplicateRemover implements DuplicateRemover {

private Set<String> urls = Sets.newSetFromMap(new ConcurrentHashMap<String, Boolean>());

@Override
public boolean isDuplicate(Request request, Task task) {
return !urls.add(getUrl(request));
}

protected String getUrl(Request request) {
return request.getUrl();
}

@Override
public void resetDuplicateCheck(Task task) {
urls.clear();
}

@Override
public int getTotalRequestsCount(Task task) {
return urls.size();
}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
DuplicateRemovedScheduler抽象类有四个具体实现类QueueScheduler，PriorityScheduler，FileCacheQueueScheduler和RedisScheduler。默认使用QueueScheduler

@ThreadSafe
public class QueueScheduler extends DuplicateRemovedScheduler implements MonitorableScheduler {

private BlockingQueue<Request> queue = new LinkedBlockingQueue<Request>();

@Override
public void pushWhenNoDuplicate(Request request, Task task) {
queue.add(request);
}

@Override
public synchronized Request poll(Task task) {
return queue.poll();
}

@Override
public int getLeftRequestsCount(Task task) {
return queue.size();
}

@Override
public int getTotalRequestsCount(Task task) {
return getDuplicateRemover().getTotalRequestsCount(task);
}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
其内部是使用了一个LinkedBlockingQueue这个无界队列来存储Request，我们应该看到了@ThreadSafe注解，那我抛一个问题吧。Scheduler是否存在线程同步问题呢，如果存在那是如何解决的呢？
再来看下一个

@ThreadSafe
public class PriorityScheduler extends DuplicateRemovedScheduler implements MonitorableScheduler {

public static final int INITIAL_CAPACITY = 5;

private BlockingQueue<Request> noPriorityQueue = new LinkedBlockingQueue<Request>();

private PriorityBlockingQueue<Request> priorityQueuePlus = new PriorityBlockingQueue<Request>(INITIAL_CAPACITY, new Comparator<Request>() {
@Override
public int compare(Request o1, Request o2) {
return -NumberUtils.compareLong(o1.getPriority(), o2.getPriority());
}
});

private PriorityBlockingQueue<Request> priorityQueueMinus = new PriorityBlockingQueue<Request>(INITIAL_CAPACITY, new Comparator<Request>() {
@Override
public int compare(Request o1, Request o2) {
return -NumberUtils.compareLong(o1.getPriority(), o2.getPriority());
}
});

@Override
public void pushWhenNoDuplicate(Request request, Task task) {
if (request.getPriority() == 0) {
noPriorityQueue.add(request);
} else if (request.getPriority() > 0) {
priorityQueuePlus.put(request);
} else {
priorityQueueMinus.put(request);
}
}

@Override
public synchronized Request poll(Task task) {
Request poll = priorityQueuePlus.poll();
if (poll != null) {
return poll;
}
poll = noPriorityQueue.poll();
if (poll != null) {
return poll;
}
return priorityQueueMinus.poll();
}

@Override
public int getLeftRequestsCount(Task task) {
return noPriorityQueue.size();
}

@Override
public int getTotalRequestsCount(Task task) {
return getDuplicateRemover().getTotalRequestsCount(task);
}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
我们看到了两个PriorityBlockingQueue和一个LinkedBlockingQueue。在poll的时候存在一个顺序。
继续

public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implements MonitorableScheduler {

private String filePath = System.getProperty("java.io.tmpdir");

private String fileUrlAllName = ".urls.txt";

private Task task;

private String fileCursor = ".cursor.txt";

private PrintWriter fileUrlWriter;

private PrintWriter fileCursorWriter;

private AtomicInteger cursor = new AtomicInteger();

private AtomicBoolean inited = new AtomicBoolean(false);

private BlockingQueue<Request> queue;

private Set<String> urls;

public FileCacheQueueScheduler(String filePath) {
if (!filePath.endsWith("/") && !filePath.endsWith("\\")) {
filePath += "/";
}
this.filePath = filePath;
}

private void flush() {
fileUrlWriter.flush();
fileCursorWriter.flush();
}

private void init(Task task) {
this.task = task;
File file = new File(filePath);
if (!file.exists()) {
file.mkdirs();
}
readFile();
initWriter();
initFlushThread();
inited.set(true);
logger.info("init cache scheduler success");
}

private void initFlushThread() {
Executors.newScheduledThreadPool(1).scheduleAtFixedRate(new Runnable() {
@Override
public void run() {
flush();
}
}, 10, 10, TimeUnit.SECONDS);
}

private void initWriter() {
try {
fileUrlWriter = new PrintWriter(new FileWriter(getFileName(fileUrlAllName), true));
fileCursorWriter = new PrintWriter(new FileWriter(getFileName(fileCursor), false));
} catch (IOException e) {
throw new RuntimeException("init cache scheduler error", e);
}
}

private void readFile() {
try {
queue = new LinkedBlockingQueue<Request>();
urls = new LinkedHashSet<String>();
readCursorFile();
readUrlFile();
} catch (FileNotFoundException e) {
//init
logger.info("init cache file " + getFileName(fileUrlAllName));
} catch (IOException e) {
logger.error("init file error", e);
}
}

private void readUrlFile() throws IOException {
String line;
BufferedReader fileUrlReader = null;
try {
fileUrlReader = new BufferedReader(new FileReader(getFileName(fileUrlAllName)));
int lineReaded = 0;
while ((line = fileUrlReader.readLine()) != null) {
urls.add(line.trim());
lineReaded++;
if (lineReaded > cursor.get()) {
queue.add(new Request(line));
}
}
} finally {
if (fileUrlReader != null) {
IOUtils.closeQuietly(fileUrlReader);
}
}
}

private void readCursorFile() throws IOException {
BufferedReader fileCursorReader = null;
try {
fileCursorReader = new BufferedReader(new FileReader(getFileName(fileCursor)));
String line;
//read the last number
while ((line = fileCursorReader.readLine()) != null) {
cursor = new AtomicInteger(NumberUtils.toInt(line));
}
} finally {
if (fileCursorReader != null) {
IOUtils.closeQuietly(fileCursorReader);
}
}
}

private String getFileName(String filename) {
return filePath + task.getUUID() + filename;
}

@Override
protected void pushWhenNoDuplicate(Request request, Task task) {
if (!inited.get()) {
init(task);
}
queue.add(request);
fileUrlWriter.println(request.getUrl());
}

@Override
public synchronized Request poll(Task task) {
if (!inited.get()) {
init(task);
}
fileCursorWriter.println(cursor.incrementAndGet());
return queue.poll();
}

@Override
public int getLeftRequestsCount(Task task) {
return queue.size();
}

@Override
public int getTotalRequestsCount(Task task) {
return getDuplicateRemover().getTotalRequestsCount(task);
}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
会将url和已经执行的url指针存在两个文件中，创建了scheduleExecutor定期的flush，所有内存中的url还是存在BlockingQueue中。
RedisScheduler不是很懂。。目前还没有接触过：）

使用

具体使用过程还是需要自己根据自己的爬虫特点然后选择特定的Scheduler及DuplicateRemover，只有懂得其原理才能选择最合适的组件。
WebMagic组件都可以自行设置这点真的太棒了~

时间： 2024-10-03 23:16:05

Scheduler的相关文章

kubernetes Master部署之Scheduler 以及 HA部署(5)

Kubernetes Scheduler作用是将Controller Manager将要新建的Pod按照特定的调度算法和调度策略绑定到集群中某个合适的Node上,并将绑定信息写入到etcd中. 一.部署Scheduler 下面生成kube-scheduler的kubeconfig文件,操作如下: cd /etc/kubernetes export KUBE_APISERVER="https://192.168.15.200:6443" 配置 cluster kubectl config

Yarn 调度器Scheduler详解

理想情况下,我们应用对Yarn资源的请求应该立刻得到满足,但现实情况资源往往是有限的,特别是在一个很繁忙的集群,一个应用资源的请求经常需要等待一段时间才能的到相应的资源.在Yarn中,负责给应用分配资源的就是Scheduler.其实调度本身就是一个难题,很难找到一个完美的策略可以解决所有的应用场景.为此,Yarn提供了多种调度器和可配置的策略供我们选择. 一.调度器的选择 在Yarn中有三种调度器可以选择:FIFO Scheduler ,Capacity Scheduler,FairS ched

当Scheduler拿不到url的 时候，不能立即退出

在webmagic的多线程抓取中有一个比较麻烦的问题:当Scheduler拿不到url的 时候,不能立即退出,需要等到没抓完的线程都运行完毕,没有新url产生时,才能退出.之前使用Thread.sleep来实现,当拿不到url 时,sleep一段时间再取,确定没有线程执行之后,再退出. 但是这种方式始终不够优雅.Java里面有wait/notify机制可以解决这种同步的问题.于是webmagic 0.4.0用wait/notify机制代替了之前的Thread.sleep机制.代码如下: whil

quick cocos scheduler update

local scheduler = require(cc.PACKAGE_NAME .. ".scheduler") local MainScene = class("MainScene", function() return display.newScene("MainScene") end) function MainScene:ctor() self:addNodeEventListener(cc.NODE_ENTER_FRAME_EVEN

Oracle Scheduler Chain 的用法

本人是借荐他人的文章目的是学习dbms_scheduler创建CHAIN类型的JOB,至于EXECUTABLE类型的JOB我己经在<DBMS_SCHEDULER执行PERL脚本加载数据>介绍了.下面转网上相关介绍: Oracle CHAIN,也就是链,是将一串需要完成的作业连在一起,根据每一个步骤完成的不同结果来确定下面的哪一个动作需要被完成.实际上也就是if .. then ..else.比如,可能有如下的定义,step 1成功执行,执行step 2,如果step 1执行失败,执行step

Linux I/O scheduler for solid-state drives

An I/O scheduler and a method for scheduling I/O requests to a solid-state drive (SSD) is disclosed. The I/O scheduler in accordance with the present disclosure bundles the write requests in such a form that the write requests in each bundle goes int

Cocos2d-x 3.1 Director ActionManger Scheduler初步分析

Director游戏主循环显示Node DisplayLinkDirector继承Director override了以下方法 virtual void mainLoop() override; virtual void setAnimationInterval(double value) override; virtual void startAnimation() override; virtual void stopAnimation() override; mainLoop()是游戏主循

Yarn调度器负载模拟器——Yarn Scheduler Load Simulator (SLS)

一.概述: Yarn调度器有许多实现,如Fifo, Capacity和Fair schedulers等.与其同时,正在进行一些优化措施来提高调度器在不同负载和工作场景下的性能.每个调度器都有自己的特性,调度决策受许多因素影响,如公平行.计算能力保证和资源的可靠性等.在部署一个调度算法到生产集群之前,评估一个调度算法是非常重要的,不幸的是,评估一个调度算法是不那么容易的,评估一个真正的集群是非常耗费时间和成本的,并且很难找到一个足够大的集群用来评测.所以,一个可以模拟在这种工作场景和负载下调度器的

Oracle数据库——Scheduler Job

日常的运维工作中,我们经常使用Linux Server的anacron服务来使得服务器执行一下计划之内的任务,可以按照特定的时间间隔,重复的执行相关的命令或者相关的脚本,来完成预期的目标,能够节省相关的人力,使得运维变得更加容易. 对于Oracle数据库来说,当然也提供了相关的功能来实现定时的,重复的完成PL/SQL Block,Shell Scripts(其实是External Executables,在这里简单用shell脚本代替),以及Oracle本身已经编写好的Storage Proce

Spark执行样例报警告：WARN scheduler.TaskSchedulerImpl: Initial job has not accepted any resources

搭建Spark环境后,调测Spark样例时,出现下面的错误:WARN scheduler.TaskSchedulerImpl: Initial job has not accepted any resources; check your cluster UI to ensure that workers are registered and have sufficient resources [[email protected] bin]$ ./run-example org.apache.sp