1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
|
<?php
/**
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License along
* with this program; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
* http://www.gnu.org/copyleft/gpl.html
*
* @file
*/
// @codeCoverageIgnoreStart
require_once __DIR__ . '/Maintenance.php';
// @codeCoverageIgnoreEnd
use MediaWiki\MainConfigNames;
use MediaWiki\Maintenance\Maintenance;
use Wikimedia\Timestamp\ConvertibleTimestamp;
/**
* Remove expired objects from the parser cache database.
*
* By default, this does not need to be run. The default parser cache
* backend is CACHE_DB (SqlBagOStuff), and by default that automatically
* performs incremental purges in the background of write requests.
*
* @see {@link MediaWiki\MainConfigSchema::ParserCacheType}
* @ingroup Maintenance
*/
class PurgeParserCache extends Maintenance {
/** @var null|string */
private $lastProgress;
/** @var null|float */
private $lastTimestamp;
/** @var int */
private $tmpCount = 0;
/** @var float */
private $usleep = 0;
public function __construct() {
parent::__construct();
$this->addDescription( "Remove old objects from the parser cache. " .
"This only works when the parser cache is in an SQL database." );
$this->addOption( 'expiredate', 'Delete objects expiring before this date.', false, true );
$this->addOption(
'age',
'Delete objects created more than this many seconds ago, assuming ' .
'$wgParserCacheExpireTime has remained consistent.',
false,
true );
$this->addOption( 'dry-run', 'Perform a dry run, to verify age and date calculation.' );
$this->addOption( 'msleep', 'Milliseconds to sleep between purge chunks of $wgUpdateRowsPerQuery.',
false,
true );
$this->addOption(
'tag',
'Purge a single server only. This feature is designed for use by large wiki farms where ' .
'one has to purge multiple servers concurrently in order to keep up with new writes. ' .
'This requires using the SqlBagOStuff "servers" option in $wgObjectCaches.',
false,
true );
}
public function execute() {
$inputDate = $this->getOption( 'expiredate' );
$inputAge = $this->getOption( 'age' );
if ( $inputDate !== null ) {
$timestamp = strtotime( $inputDate );
} elseif ( $inputAge !== null ) {
$expireTime = (int)$this->getConfig()->get( MainConfigNames::ParserCacheExpireTime );
$timestamp = time() + $expireTime - intval( $inputAge );
} else {
$this->fatalError( "Must specify either --expiredate or --age" );
}
$this->usleep = 1e3 * $this->getOption( 'msleep', 0 );
$this->lastTimestamp = microtime( true );
$humanDate = ConvertibleTimestamp::convert( TS_RFC2822, $timestamp );
if ( $this->hasOption( 'dry-run' ) ) {
$this->fatalError( "\nDry run mode, would delete objects having an expiry before " . $humanDate . "\n" );
}
$this->output( "Deleting objects expiring before " . $humanDate . "\n" );
$pc = $this->getServiceContainer()->getParserCache()->getCacheStorage();
$success = $pc->deleteObjectsExpiringBefore(
$timestamp,
[ $this, 'showProgressAndWait' ],
INF,
// Note that "0" can be a valid server tag, and must not be discarded or changed to null.
$this->getOption( 'tag', null )
);
if ( !$success ) {
$this->fatalError( "\nCannot purge this kind of parser cache." );
}
$this->showProgressAndWait( 100 );
$this->output( "\nDone\n" );
}
public function showProgressAndWait( $percent ) {
// Parser caches involve mostly-unthrottled writes of large blobs. This is sometimes prone
// to replication lag. As such, while our purge queries are simple primary key deletes,
// we want to avoid adding significant load to the replication stream, by being
// proactively graceful with these sleeps between each batch.
// The reason we don't explicitly wait for replication is that that would require the script
// to be aware of cross-dc replicas, which we prefer not to, and waiting for replication
// and confirmation latency might actually be *too* graceful and take so long that the
// purge script would not be able to finish within 24 hours for large wiki farms.
// (T150124).
usleep( $this->usleep );
$this->tmpCount++;
$percentString = sprintf( "%.1f", $percent );
if ( $percentString === $this->lastProgress ) {
// Only print a line if we've progressed >= 0.1% since the last printed line.
// This does not mean every 0.1% step is printed since we only run this callback
// once after a deletion batch. How often and how many lines we print depends on the
// batch size (SqlBagOStuff::deleteObjectsExpiringBefore, $wgUpdateRowsPerQuery),
// and on how many table rows there are.
return;
}
$now = microtime( true );
$sec = sprintf( "%.1f", $now - $this->lastTimestamp );
// Give a sense of how much time is spent in the delete operations vs the sleep time,
// by recording the number of iterations we've completed since the last progress update.
$this->output( "... {$percentString}% done (+{$this->tmpCount} iterations in {$sec}s)\n" );
$this->lastProgress = $percentString;
$this->tmpCount = 0;
$this->lastTimestamp = $now;
}
}
// @codeCoverageIgnoreStart
$maintClass = PurgeParserCache::class;
require_once RUN_MAINTENANCE_IF_MAIN;
// @codeCoverageIgnoreEnd
|
