1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
|
<?php
/**
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
*/
namespace MediaWiki\PoolCounter;
use MediaWiki\Status\Status;
use Psr\Log\LoggerAwareInterface;
use Psr\Log\LoggerInterface;
use Psr\Log\NullLogger;
/**
* Semaphore semantics to restrict how many workers may concurrently perform a task.
*
* When you have many workers (threads/servers) in service, and a
* cached item expensive to produce expires, you may get several workers
* computing the same expensive item at the same time.
*
* Given enough incoming requests and the item expiring quickly (non-cacheable,
* or lots of edits or other invalidation events) that single task can end up
* unfairly using most (or all) of the CPUs of the server cluster.
* This is also known as "Michael Jackson effect", as this scenario happened on
* the English Wikipedia in 2009 on the day Michael Jackson died.
* See also <https://wikitech.wikimedia.org/wiki/Michael_Jackson_effect>.
*
* PoolCounter was created to provide semaphore semantics to restrict the number
* of workers that may be concurrently performing a given task. Only one key
* can be locked by any PoolCounter instance of a process, except for keys
* that start with "nowait:". However, only non-blocking requests (timeout=0)
* may be used with a "nowait:" key.
*
* By default PoolCounterNull is used, which provides no locking.
* Install the poolcounterd service from
* <https://gerrit.wikimedia.org/g/mediawiki/services/poolcounter> to
* enable this feature.
*
* @since 1.16
* @stable to extend
*/
abstract class PoolCounter implements LoggerAwareInterface {
/* Return codes */
public const LOCKED = 1; /* Lock acquired */
public const RELEASED = 2; /* Lock released */
public const DONE = 3; /* Another worker did the work for you */
public const ERROR = -1; /* Indeterminate error */
public const NOT_LOCKED = -2; /* Called release() with no lock held */
public const QUEUE_FULL = -3; /* There are already maxqueue workers on this lock */
public const TIMEOUT = -4; /* Timeout exceeded */
public const LOCK_HELD = -5; /* Cannot acquire another lock while you have one lock held */
/** @var string All workers with the same key share the lock */
protected $key;
/** @var int Maximum number of workers working on tasks with the same key simultaneously */
protected $workers;
/**
* Maximum number of workers working on this task type, regardless of key.
* 0 means unlimited. Max allowed value is 65536.
* The way the slot limit is enforced is overzealous - this option should be used with caution.
* @var int
*/
protected $slots = 0;
/** @var int If this number of workers are already working/waiting, fail instead of wait */
protected $maxqueue;
/** @var int Maximum time in seconds to wait for the lock */
protected $timeout;
protected LoggerInterface $logger;
/**
* @var bool Whether the key is a "might wait" key
*/
private $isMightWaitKey;
/**
* @var int Whether this process holds a "might wait" lock key
*/
private static $acquiredMightWaitKey = 0;
/**
* @var bool Enable fast stale mode (T250248). This may be overridden by the work class.
*/
private $fastStale;
/**
* @param array $conf
* @param string $type The class of actions to limit concurrency for (task type)
* @param string $key
*/
public function __construct( array $conf, string $type, string $key ) {
$this->workers = $conf['workers'];
$this->maxqueue = $conf['maxqueue'];
$this->timeout = $conf['timeout'];
if ( isset( $conf['slots'] ) ) {
$this->slots = $conf['slots'];
}
$this->fastStale = $conf['fastStale'] ?? false;
$this->logger = new NullLogger();
if ( $this->slots ) {
$key = $this->hashKeyIntoSlots( $type, $key, $this->slots );
}
$this->key = $key;
$this->isMightWaitKey = !preg_match( '/^nowait:/', $this->key );
}
/**
* @return string
*/
public function getKey() {
return $this->key;
}
/**
* I want to do this task and I need to do it myself.
*
* @param int|null $timeout Wait timeout, or null to use value passed to
* the constructor
* @return Status Value is one of Locked/Error
*/
abstract public function acquireForMe( $timeout = null );
/**
* I want to do this task, but if anyone else does it
* instead, it's also fine for me. I will read its cached data.
*
* @param int|null $timeout Wait timeout, or null to use value passed to
* the constructor
* @return Status Value is one of Locked/Done/Error
*/
abstract public function acquireForAnyone( $timeout = null );
/**
* I have successfully finished my task.
* Lets another one grab the lock, and returns the workers
* waiting on acquireForAnyone()
*
* @return Status Value is one of Released/NotLocked/Error
*/
abstract public function release();
/**
* Checks that the lock request is sensible.
* @return Status good for sensible requests, fatal for the not so sensible
* @since 1.25
*/
final protected function precheckAcquire() {
if ( $this->isMightWaitKey ) {
if ( self::$acquiredMightWaitKey ) {
/*
* The poolcounter itself is quite happy to allow you to wait
* on another lock while you have a lock you waited on already
* but we think that it is unlikely to be a good idea. So we
* made it an error. If you are _really_ _really_ sure it is a
* good idea then feel free to implement an unsafe flag or
* something.
*/
return Status::newFatal(
'poolcounter-usage-error',
'You may only aquire a single non-nowait lock.'
);
}
} elseif ( $this->timeout !== 0 ) {
return Status::newFatal(
'poolcounter-usage-error',
'Locks starting in nowait: must have 0 timeout.'
);
}
return Status::newGood();
}
/**
* Update any lock tracking information when the lock is acquired
* @since 1.25
*/
final protected function onAcquire() {
self::$acquiredMightWaitKey |= $this->isMightWaitKey;
}
/**
* Update any lock tracking information when the lock is released
* @since 1.25
*/
final protected function onRelease() {
self::$acquiredMightWaitKey &= !$this->isMightWaitKey;
}
/**
* Given a key (any string) and the number of lots, returns a slot key (a prefix with a suffix
* integer from the [0..($slots-1)] range). This is used for a global limit on the number of
* instances of a given type that can acquire a lock. The hashing is deterministic so that
* PoolCounter::$workers is always an upper limit of how many instances with the same key
* can acquire a lock.
*
* @param string $type The class of actions to limit concurrency for (task type)
* @param string $key PoolCounter instance key (any string)
* @param int $slots The number of slots (max allowed value is 65536)
* @return string Slot key with the type and slot number
*/
protected function hashKeyIntoSlots( $type, $key, $slots ) {
return $type . ':' . ( hexdec( substr( sha1( $key ), 0, 4 ) ) % $slots );
}
/**
* Is fast stale mode (T250248) enabled? This may be overridden by the
* PoolCounterWork subclass.
*
* @return bool
*/
public function isFastStaleEnabled() {
return $this->fastStale;
}
/**
* @since 1.42
* @param LoggerInterface $logger
* @return void
*/
public function setLogger( LoggerInterface $logger ) {
$this->logger = $logger;
}
/**
* @internal For use in PoolCounterWork only
* @return LoggerInterface
*/
public function getLogger(): LoggerInterface {
return $this->logger;
}
}
/**
* @deprecated since 1.42
*/
class_alias( PoolCounter::class, 'PoolCounter' );
|
