resourceloader: support tracking indirect module dependency paths via BagOStuff

This can be enabled via a configuration flag. Otherwise, SqlModuleDependencyStore
will be used in order to keep using the module_deps table.

Create a dependency store class, wrapping BagOStuff, that stores known module
dependencies. Inject it into ResourceLoader and inject the path lists into
ResourceLoaderModule directly and via callback.

Bug: T113916
Change-Id: I6da55e78d5554e30e5df6b4bc45d84817f5bea15

7 files changed, 623 insertions, 139 deletions

diff --git a/includes/resourceloader/ResourceLoader.php b/includes/resourceloader/ResourceLoader.php
index 1dda454e354b..baa0ce877694 100644
--- a/includes/resourceloader/ResourceLoader.php
+++ b/includes/resourceloader/ResourceLoader.php
@@ -20,10 +20,13 @@
  * @author Trevor Parscal
  */
 
+use MediaWiki\HeaderCallback;
 use MediaWiki\MediaWikiServices;
 use Psr\Log\LoggerAwareInterface;
 use Psr\Log\LoggerInterface;
 use Psr\Log\NullLogger;
+use Wikimedia\DependencyStore\DependencyStore;
+use Wikimedia\DependencyStore\KeyValueDependencyStore;
 use Wikimedia\Rdbms\DBConnectionError;
 use Wikimedia\Timestamp\ConvertibleTimestamp;
 use Wikimedia\WrappedString;
@@ -47,6 +50,8 @@ class ResourceLoader implements LoggerAwareInterface {
 	protected $config;
 	/** @var MessageBlobStore */
 	protected $blobStore;
+	/** @var DependencyStore */
+	protected $depStore;
 
 	/** @var LoggerInterface */
 	private $logger;
@@ -63,7 +68,6 @@ class ResourceLoader implements LoggerAwareInterface {
 	protected $testModuleNames = [];
 	/** @var string[] List of module names that contain QUnit test suites */
 	protected $testSuiteModuleNames = [];
-
 	/** @var array Map of (source => path); E.g. [ 'source-id' => 'http://.../load.php' ] */
 	protected $sources = [];
 	/** @var array Errors accumulated during current respond() call */
@@ -71,79 +75,69 @@ class ResourceLoader implements LoggerAwareInterface {
 	/** @var string[] Extra HTTP response headers from modules loaded in makeModuleResponse() */
 	protected $extraHeaders = [];
 
+	/** @var array Map of (module-variant => buffered DependencyStore updates) */
+	private $depStoreUpdateBuffer = [];
+
 	/** @var bool */
 	protected static $debugMode = null;
 
 	/** @var int */
 	const CACHE_VERSION = 8;
 
+	/** @var string */
+	private const RL_DEP_STORE_PREFIX = 'ResourceLoaderModule';
+	/** @var int Expiry (in seconds) of indirect dependency information for modules */
+	private const RL_MODULE_DEP_TTL = BagOStuff::TTL_WEEK;
+
 	/** @var string JavaScript / CSS pragma to disable minification. * */
 	const FILTER_NOMIN = '/*@nomin*/';
 
 	/**
-	 * Load information stored in the database about modules.
+	 * Load information stored in the database and dependency tracking store about modules
 	 *
-	 * This method grabs modules dependencies from the database and updates modules
-	 * objects.
-	 *
-	 * This is not inside the module code because it is much faster to
-	 * request all of the information at once than it is to have each module
-	 * requests its own information. This sacrifice of modularity yields a substantial
-	 * performance improvement.
-	 *
-	 * @param array $moduleNames List of module names to preload information for
-	 * @param ResourceLoaderContext $context Context to load the information within
+	 * @param string[] $moduleNames Module names
+	 * @param ResourceLoaderContext $context ResourceLoader-specific context of the request
 	 */
 	public function preloadModuleInfo( array $moduleNames, ResourceLoaderContext $context ) {
-		if ( !$moduleNames ) {
-			// Or else Database*::select() will explode, plus it's cheaper!
-			return;
-		}
-		$dbr = wfGetDB( DB_REPLICA );
-		$lang = $context->getLanguage();
-
-		// Batched version of ResourceLoaderModule::getFileDependencies
+		// Load all tracked indirect file dependencies for the modules
 		$vary = ResourceLoaderModule::getVary( $context );
-		$res = $dbr->select( 'module_deps', [ 'md_module', 'md_deps' ], [
-				'md_module' => $moduleNames,
-				'md_skin' => $vary,
-			], __METHOD__
-		);
-
-		// Prime in-object cache for file dependencies
-		$modulesWithDeps = [];
-		foreach ( $res as $row ) {
-			$module = $this->getModule( $row->md_module );
-			if ( $module ) {
-				$module->setFileDependencies( $context, ResourceLoaderModule::expandRelativePaths(
-					json_decode( $row->md_deps, true )
-				) );
-				$modulesWithDeps[] = $row->md_module;
-			}
+		$entitiesByModule = [];
+		foreach ( $moduleNames as $moduleName ) {
+			$entitiesByModule[$moduleName] = "$moduleName|$vary";
 		}
-		// Register the absence of a dependency row too
-		foreach ( array_diff( $moduleNames, $modulesWithDeps ) as $name ) {
-			$module = $this->getModule( $name );
+		$depsByEntity = $this->depStore->retrieveMulti(
+			self::RL_DEP_STORE_PREFIX,
+			$entitiesByModule
+		);
+		// Inject the indirect file dependencies for all the modules
+		foreach ( $moduleNames as $moduleName ) {
+			$module = $this->getModule( $moduleName );
 			if ( $module ) {
-				$module->setFileDependencies( $context, [] );
+				$entity = $entitiesByModule[$moduleName];
+				$deps = $depsByEntity[$entity];
+				$paths = ResourceLoaderModule::expandRelativePaths( $deps['paths'] );
+				$module->setFileDependencies( $context, $paths );
 			}
 		}
 
 		// Batched version of ResourceLoaderWikiModule::getTitleInfo
+		$dbr = wfGetDB( DB_REPLICA );
 		ResourceLoaderWikiModule::preloadTitleInfo( $context, $dbr, $moduleNames );
 
 		// Prime in-object cache for message blobs for modules with messages
-		$modules = [];
-		foreach ( $moduleNames as $name ) {
-			$module = $this->getModule( $name );
+		$modulesWithMessages = [];
+		foreach ( $moduleNames as $moduleName ) {
+			$module = $this->getModule( $moduleName );
 			if ( $module && $module->getMessages() ) {
-				$modules[$name] = $module;
+				$modulesWithMessages[$moduleName] = $module;
 			}
 		}
+		// Prime in-object cache for message blobs for modules with messages
+		$lang = $context->getLanguage();
 		$store = $this->getMessageBlobStore();
-		$blobs = $store->getBlobs( $modules, $lang );
-		foreach ( $blobs as $name => $blob ) {
-			$modules[$name]->setMessageBlob( $blob, $lang );
+		$blobs = $store->getBlobs( $modulesWithMessages, $lang );
+		foreach ( $blobs as $moduleName => $blob ) {
+			$modulesWithMessages[$moduleName]->setMessageBlob( $blob, $lang );
 		}
 	}
 
@@ -218,8 +212,13 @@ class ResourceLoader implements LoggerAwareInterface {
 	 * Register core modules and runs registration hooks.
 	 * @param Config|null $config
 	 * @param LoggerInterface|null $logger [optional]
+	 * @param DependencyStore|null $tracker [optional]
 	 */
-	public function __construct( Config $config = null, LoggerInterface $logger = null ) {
+	public function __construct(
+		Config $config = null,
+		LoggerInterface $logger = null,
+		DependencyStore $tracker = null
+	) {
 		$this->logger = $logger ?: new NullLogger();
 		$services = MediaWikiServices::getInstance();
 
@@ -238,6 +237,9 @@ class ResourceLoader implements LoggerAwareInterface {
 		$this->setMessageBlobStore(
 			new MessageBlobStore( $this, $this->logger, $services->getMainWANObjectCache() )
 		);
+
+		$tracker = $tracker ?: new KeyValueDependencyStore( new HashBagOStuff() );
+		$this->setDependencyStore( $tracker );
 	}
 
 	/**
@@ -280,6 +282,14 @@ class ResourceLoader implements LoggerAwareInterface {
 	}
 
 	/**
+	 * @since 1.35
+	 * @param DependencyStore $tracker
+	 */
+	public function setDependencyStore( DependencyStore $tracker ) {
+		$this->depStore = $tracker;
+	}
+
+	/**
 	 * Register a module with the ResourceLoader system.
 	 *
 	 * @param string|array[] $name Module name as a string or, array of module info arrays
@@ -506,6 +516,10 @@ class ResourceLoader implements LoggerAwareInterface {
 			$object->setConfig( $this->getConfig() );
 			$object->setLogger( $this->logger );
 			$object->setName( $name );
+			$object->setDependencyAccessCallbacks(
+				[ $this, 'loadModuleDependenciesInternal' ],
+				[ $this, 'saveModuleDependenciesInternal' ]
+			);
 			$this->modules[$name] = $object;
 		}
 
@@ -513,6 +527,74 @@ class ResourceLoader implements LoggerAwareInterface {
 	}
 
 	/**
+	 * @param string $moduleName Module name
+	 * @param string $variant Language/skin variant
+	 * @return string[] List of absolute file paths
+	 * @private
+	 */
+	public function loadModuleDependenciesInternal( $moduleName, $variant ) {
+		$deps = $this->depStore->retrieve( self::RL_DEP_STORE_PREFIX, "$moduleName|$variant" );
+
+		return ResourceLoaderModule::expandRelativePaths( $deps['paths'] );
+	}
+
+	/**
+	 * @param string $moduleName Module name
+	 * @param string $variant Language/skin variant
+	 * @param string[] $paths List of relative paths referenced during computation
+	 * @param string[] $priorPaths List of relative paths tracked in the dependency store
+	 * @private
+	 */
+	public function saveModuleDependenciesInternal( $moduleName, $variant, $paths, $priorPaths ) {
+		$hasPendingUpdate = (bool)$this->depStoreUpdateBuffer;
+		$entity = "$moduleName|$variant";
+
+		if ( array_diff( $paths, $priorPaths ) || array_diff( $priorPaths, $paths ) ) {
+			// Dependency store needs to be updated with the new path list
+			if ( $paths ) {
+				$deps = $this->depStore->newEntityDependencies( $paths, time() );
+				$this->depStoreUpdateBuffer[$entity] = $deps;
+			} else {
+				$this->depStoreUpdateBuffer[$entity] = null;
+			}
+		} elseif ( $priorPaths ) {
+			// Dependency store needs to store the existing path list for longer
+			$this->depStoreUpdateBuffer[$entity] = '*';
+		}
+
+		// Use a DeferrableUpdate to flush the buffered dependency updates...
+		if ( !$hasPendingUpdate ) {
+			DeferredUpdates::addCallableUpdate( function () {
+				$updatesByEntity = $this->depStoreUpdateBuffer;
+				$this->depStoreUpdateBuffer = []; // consume
+				$cache = ObjectCache::getLocalClusterInstance();
+
+				$scopeLocks = [];
+				$depsByEntity = [];
+				$entitiesUnreg = [];
+				$entitiesRenew = [];
+				foreach ( $updatesByEntity as $entity => $update ) {
+					$scopeLocks[$entity] = $cache->getScopedLock( "rl-deps:$entity", 0 );
+					if ( !$scopeLocks[$entity] ) {
+						continue; // avoid duplicate write request slams (T124649)
+					} elseif ( $update === null ) {
+						$entitiesUnreg[] = $entity;
+					} elseif ( $update === '*' ) {
+						$entitiesRenew[] = $entity;
+					} else {
+						$depsByEntity[$entity] = $update;
+					}
+				}
+
+				$ttl = self::RL_MODULE_DEP_TTL;
+				$this->depStore->storeMulti( self::RL_DEP_STORE_PREFIX, $depsByEntity, $ttl );
+				$this->depStore->remove( self::RL_DEP_STORE_PREFIX, $entitiesUnreg );
+				$this->depStore->renew( self::RL_DEP_STORE_PREFIX, $entitiesRenew, $ttl );
+			} );
+		}
+	}
+
+	/**
 	 * Whether the module is a ResourceLoaderFileModule (including subclasses).
 	 *
 	 * @param string $name Module name
@@ -865,7 +947,7 @@ class ResourceLoader implements LoggerAwareInterface {
 	protected function sendResponseHeaders(
 		ResourceLoaderContext $context, $etag, $errors, array $extra = []
 	) {
-		\MediaWiki\HeaderCallback::warnIfHeadersSent();
+		HeaderCallback::warnIfHeadersSent();
 		$rlMaxage = $this->config->get( 'ResourceLoaderMaxage' );
 		// Use a short cache expiry so that updates propagate to clients quickly, if:
 		// - No version specified (shared resources, e.g. stylesheets)
diff --git a/includes/resourceloader/ResourceLoaderFileModule.php b/includes/resourceloader/ResourceLoaderFileModule.php
index b62d83955422..b1bfac6146d6 100644
--- a/includes/resourceloader/ResourceLoaderFileModule.php
+++ b/includes/resourceloader/ResourceLoaderFileModule.php
@@ -436,7 +436,9 @@ class ResourceLoaderFileModule extends ResourceLoaderModule {
 			$this->getStyleFiles( $context ),
 			$context
 		);
-		// Collect referenced files
+
+		// Track indirect file dependencies so that ResourceLoaderStartUpModule can check for
+		// on-disk file changes to any of this files without having to recompute the file list
 		$this->saveFileDependencies( $context, $this->localFileRefs );
 
 		return $styles;
@@ -527,7 +529,6 @@ class ResourceLoaderFileModule extends ResourceLoaderModule {
 	/**
 	 * Helper method for getDefinitionSummary.
 	 *
-	 * @see ResourceLoaderModule::getFileDependencies
 	 * @param ResourceLoaderContext $context
 	 * @return string
 	 */
diff --git a/includes/resourceloader/ResourceLoaderModule.php b/includes/resourceloader/ResourceLoaderModule.php
index 27f8b0e4c0f6..6afaafaeb1fd 100644
--- a/includes/resourceloader/ResourceLoaderModule.php
+++ b/includes/resourceloader/ResourceLoaderModule.php
@@ -26,7 +26,6 @@ use Psr\Log\LoggerInterface;
 use Psr\Log\NullLogger;
 use Wikimedia\AtEase\AtEase;
 use Wikimedia\RelPath;
-use Wikimedia\ScopedCallback;
 
 /**
  * Abstraction for ResourceLoader modules, with name registration and maxage functionality.
@@ -63,6 +62,11 @@ abstract class ResourceLoaderModule implements LoggerAwareInterface {
 	/** @var array Map of (context hash => cached module content) */
 	protected $contents = [];
 
+	/** @var callback Function of (module name, variant) to get indirect file dependencies */
+	private $depLoadCallback;
+	/** @var callback Function of (module name, variant) to get indirect file dependencies */
+	private $depSaveCallback;
+
 	/** @var string|bool Deprecation string or true if deprecated; false otherwise */
 	protected $deprecated = false;
 
@@ -114,6 +118,18 @@ abstract class ResourceLoaderModule implements LoggerAwareInterface {
 	}
 
 	/**
+	 * Inject the functions that load/save the indirect file path dependency list from storage
+	 *
+	 * @param callable $loadCallback Function of (module name, variant)
+	 * @param callable $saveCallback Function of (module name, variant, current paths, stored paths)
+	 * @since 1.35
+	 */
+	public function setDependencyAccessCallbacks( callable $loadCallback, callable $saveCallback ) {
+		$this->depLoadCallback = $loadCallback;
+		$this->depSaveCallback = $saveCallback;
+	}
+
+	/**
 	 * Get this module's origin. This is set when the module is registered
 	 * with ResourceLoader::register()
 	 *
@@ -390,125 +406,89 @@ abstract class ResourceLoaderModule implements LoggerAwareInterface {
 	}
 
 	/**
-	 * Get the files this module depends on indirectly for a given skin.
+	 * Get the indirect dependencies for this module persuant to the skin/language context
+	 *
+	 * These are only image files referenced by the module's stylesheet
 	 *
-	 * These are only image files referenced by the module's stylesheet.
+	 * If niether setFileDependencies() nor setDependencyLoadCallback() was called, this
+	 * will simply return a placeholder with an empty file list
+	 *
+	 * @see ResourceLoader::setFileDependencies()
+	 * @see ResourceLoader::saveFileDependencies()
 	 *
 	 * @param ResourceLoaderContext $context
-	 * @return array List of files
+	 * @return string[] List of absolute file paths
+	 * @throws RuntimeException When setFileDependencies() has not yet been called
 	 */
 	protected function getFileDependencies( ResourceLoaderContext $context ) {
-		$vary = self::getVary( $context );
-
-		// Try in-object cache first
-		if ( !isset( $this->fileDeps[$vary] ) ) {
-			$dbr = wfGetDB( DB_REPLICA );
-			$deps = $dbr->selectField( 'module_deps',
-				'md_deps',
-				[
-					'md_module' => $this->getName(),
-					'md_skin' => $vary,
-				],
-				__METHOD__
-			);
+		$variant = self::getVary( $context );
 
-			if ( $deps !== null ) {
-				$this->fileDeps[$vary] = self::expandRelativePaths(
-					(array)json_decode( $deps, true )
-				);
+		if ( !isset( $this->fileDeps[$variant] ) ) {
+			if ( $this->depLoadCallback ) {
+				$this->fileDeps[$variant] =
+					call_user_func( $this->depLoadCallback, $this->getName(), $variant );
 			} else {
-				$this->fileDeps[$vary] = [];
+				$this->getLogger()->info( __METHOD__ . ": no callback registered" );
+				$this->fileDeps[$variant] = [];
 			}
 		}
-		return $this->fileDeps[$vary];
+
+		return $this->fileDeps[$variant];
 	}
 
 	/**
-	 * Set in-object cache for file dependencies.
+	 * Set the indirect dependencies for this module persuant to the skin/language context
 	 *
-	 * This is used to retrieve data in batches. See ResourceLoader::preloadModuleInfo().
-	 * To save the data, use saveFileDependencies().
+	 * These are only image files referenced by the module's stylesheet
+	 *
+	 * @see ResourceLoader::getFileDependencies()
+	 * @see ResourceLoader::saveFileDependencies()
 	 *
 	 * @param ResourceLoaderContext $context
-	 * @param string[] $files Array of file names
+	 * @param string[] $paths List of absolute file paths
 	 */
-	public function setFileDependencies( ResourceLoaderContext $context, $files ) {
-		$vary = self::getVary( $context );
-		$this->fileDeps[$vary] = $files;
+	public function setFileDependencies( ResourceLoaderContext $context, array $paths ) {
+		$variant = self::getVary( $context );
+		$this->fileDeps[$variant] = $paths;
 	}
 
 	/**
-	 * Set the files this module depends on indirectly for a given skin.
+	 * Save the indirect dependencies for this module persuant to the skin/language context
 	 *
-	 * @since 1.27
 	 * @param ResourceLoaderContext $context
-	 * @param array $localFileRefs List of files
+	 * @param string[] $curFileRefs List of newly computed indirect file dependencies
+	 * @since 1.27
 	 */
-	protected function saveFileDependencies( ResourceLoaderContext $context, array $localFileRefs ) {
+	protected function saveFileDependencies( ResourceLoaderContext $context, array $curFileRefs ) {
+		if ( !$this->depSaveCallback ) {
+			$this->getLogger()->info( __METHOD__ . ": no callback registered" );
+
+			return;
+		}
+
 		try {
-			// Related bugs and performance considerations:
-			// 1. Don't needlessly change the database value with the same list in a
-			//    different order or with duplicates.
+			// Pitfalls and performance considerations:
+			// 1. Don't keep updating the tracked paths due to duplicates or sorting.
 			// 2. Use relative paths to avoid ghost entries when $IP changes. (T111481)
-			// 3. Don't needlessly replace the database with the same value
+			// 3. Don't needlessly replace tracked paths with the same value
 			//    just because $IP changed (e.g. when upgrading a wiki).
 			// 4. Don't create an endless replace loop on every request for this
 			//    module when '../' is used anywhere. Even though both are expanded
 			//    (one expanded by getFileDependencies from the DB, the other is
 			//    still raw as originally read by RL), the latter has not
 			//    been normalized yet.
-
-			// Normalise
-			$localFileRefs = array_values( array_unique( $localFileRefs ) );
-			sort( $localFileRefs );
-			$localPaths = self::getRelativePaths( $localFileRefs );
-			$storedPaths = self::getRelativePaths( $this->getFileDependencies( $context ) );
-
-			if ( $localPaths === $storedPaths ) {
-				// Unchanged. Avoid needless database query (especially master conn!).
-				return;
-			}
-
-			// The file deps list has changed, we want to update it.
-			$vary = self::getVary( $context );
-			$cache = ObjectCache::getLocalClusterInstance();
-			$key = $cache->makeKey( __METHOD__, $this->getName(), $vary );
-			$scopeLock = $cache->getScopedLock( $key, 0 );
-			if ( !$scopeLock ) {
-				// Another request appears to be doing this update already.
-				// Avoid write slams (T124649).
-				return;
-			}
-
-			// No needless escaping as this isn't HTML output.
-			// Only stored in the database and parsed in PHP.
-			$deps = json_encode( $localPaths, JSON_UNESCAPED_SLASHES );
-			$dbw = wfGetDB( DB_MASTER );
-			$dbw->upsert( 'module_deps',
-				[
-					'md_module' => $this->getName(),
-					'md_skin' => $vary,
-					'md_deps' => $deps,
-				],
-				[ [ 'md_module', 'md_skin' ] ],
-				[
-					'md_deps' => $deps,
-				],
-				__METHOD__
+			call_user_func(
+				$this->depSaveCallback,
+				$this->getName(),
+				self::getVary( $context ),
+				self::getRelativePaths( $curFileRefs ),
+				self::getRelativePaths( $this->getFileDependencies( $context ) )
 			);
-
-			if ( $dbw->trxLevel() ) {
-				$dbw->onTransactionResolution(
-					function () use ( &$scopeLock ) {
-						ScopedCallback::consume( $scopeLock ); // release after commit
-					},
-					__METHOD__
-				);
-			}
 		} catch ( Exception $e ) {
-			// Probably a DB failure. Either the read query from getFileDependencies(),
-			// or the write query above.
-			$this->getLogger()->error( "Failed to update DB: $e", [ 'exception' => $e ] );
+			$this->getLogger()->warning(
+				__METHOD__ . ": failed to update dependencies: {$e->getMessage()}",
+				[ 'exception' => $e ]
+			);
 		}
 	}
 
diff --git a/includes/resourceloader/dependencystore/DependencyStore.php b/includes/resourceloader/dependencystore/DependencyStore.php
new file mode 100644
index 000000000000..086d3b9a4769
--- /dev/null
+++ b/includes/resourceloader/dependencystore/DependencyStore.php
@@ -0,0 +1,128 @@
+<?php
+/**
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ */
+
+namespace Wikimedia\DependencyStore;
+
+/**
+ * Class for tracking per-entity dependency path lists that are expensive to mass compute
+ *
+ * @internal This should not be used outside of ResourceLoader and ResourceLoaderModule
+ *
+ * @since 1.35
+ */
+abstract class DependencyStore {
+	/** @var string */
+	const KEY_PATHS = 'paths';
+	/** @var string */
+	const KEY_AS_OF = 'asOf';
+
+	/**
+	 * @param string[] $paths List of dependency paths
+	 * @param int|null $asOf UNIX timestamp or null
+	 * @return array
+	 */
+	public function newEntityDependencies( array $paths = [], $asOf = null ) {
+		return [ self::KEY_PATHS => $paths, self::KEY_AS_OF => $asOf ];
+	}
+
+	/**
+	 * Get the currently tracked dependencies for an entity
+	 *
+	 * The "paths" field contains a sorted list of unique paths
+	 *
+	 * The "asOf" field reflects the last-modified timestamp of the dependency data itself.
+	 * It will be null if there is no tracking data available. Note that if empty path lists
+	 * are never stored (as an optimisation) then it will not be possible to discern whether
+	 * the result is up-to-date.
+	 *
+	 * @param string $type Entity type
+	 * @param string $entity Entity name
+	 * @return array Map of (paths: paths, asOf: UNIX timestamp or null)
+	 * @throws DependencyStoreException
+	 */
+	final public function retrieve( $type, $entity ) {
+		return $this->retrieveMulti( $type, [ $entity ] )[$entity];
+	}
+
+	/**
+	 * Get the currently tracked dependencies for a set of entities
+	 *
+	 * @see KeyValueDependencyStore::retrieve()
+	 *
+	 * @param string $type Entity type
+	 * @param string[] $entities Entity names
+	 * @return array[] Map of (entity => (paths: paths, asOf: UNIX timestamp or null))
+	 * @throws DependencyStoreException
+	 */
+	abstract public function retrieveMulti( $type, array $entities );
+
+	/**
+	 * Set the currently tracked dependencies for an entity
+	 *
+	 * Dependency data should be set to persist as long as anything might rely on it existing
+	 * in order to check the validity of some previously computed work. This can be achieved
+	 * while minimizing storage space under the following scheme:
+	 *   - a) computed work has a TTL (time-to-live)
+	 *   - b) when work is computed, the dependency data is updated
+	 *   - c) the dependency data has a TTL higher enough to accounts for skew/latency
+	 *   - d) the TTL of tracked dependency data is renewed upon access
+	 *
+	 * @param string $type Entity type
+	 * @param string $entity Entity name
+	 * @param array $data Map of (paths: paths, asOf: UNIX timestamp or null)
+	 * @param int $ttl New time-to-live in seconds
+	 * @throws DependencyStoreException
+	 */
+	final public function store( $type, $entity, array $data, $ttl ) {
+		$this->storeMulti( $type, [ $entity => $data ], $ttl );
+	}
+
+	/**
+	 * Set the currently tracked dependencies for a set of entities
+	 *
+	 * @see KeyValueDependencyStore::store()
+	 *
+	 * @param string $type Entity type
+	 * @param array[] $dataByEntity Map of (entity => (paths: paths, asOf: UNIX timestamp or null))
+	 * @param int $ttl New time-to-live in seconds
+	 * @throws DependencyStoreException
+	 *
+	 */
+	abstract public function storeMulti( $type, array $dataByEntity, $ttl );
+
+	/**
+	 * Delete the currently tracked dependencies for an entity or set of entities
+	 *
+	 * @param string $type Entity type
+	 * @param string|string[] $entities Entity name(s)
+	 * @throws DependencyStoreException
+	 */
+	abstract public function remove( $type, $entities );
+
+	/**
+	 * Set the expiry for the currently tracked dependencies for an entity or set of entities
+	 *
+	 * @param string $type Entity type
+	 * @param string|string[] $entities Entity name(s)
+	 * @param int $ttl New time-to-live in seconds
+	 * @throws DependencyStoreException
+	 */
+	abstract public function renew( $type, $entities, $ttl );
+}
diff --git a/includes/resourceloader/dependencystore/DependencyStoreException.php b/includes/resourceloader/dependencystore/DependencyStoreException.php
new file mode 100644
index 000000000000..36173549bdec
--- /dev/null
+++ b/includes/resourceloader/dependencystore/DependencyStoreException.php
@@ -0,0 +1,12 @@
+<?php
+
+namespace Wikimedia\DependencyStore;
+
+use RuntimeException;
+
+/**
+ * @since 1.35
+ */
+class DependencyStoreException extends RuntimeException {
+
+}
diff --git a/includes/resourceloader/dependencystore/KeyValueDependencyStore.php b/includes/resourceloader/dependencystore/KeyValueDependencyStore.php
new file mode 100644
index 000000000000..4c510ba92710
--- /dev/null
+++ b/includes/resourceloader/dependencystore/KeyValueDependencyStore.php
@@ -0,0 +1,118 @@
+<?php
+/**
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ */
+
+namespace Wikimedia\DependencyStore;
+
+use BagOStuff;
+use InvalidArgumentException;
+
+/**
+ * Lightweight class for tracking path dependencies lists via an object cache instance
+ *
+ * This does not throw DependencyStoreException due to I/O errors since it is optimized for
+ * speed and availability. Read methods return empty placeholders on failure. Write methods
+ * might issue I/O in the background and return immediately. However, reads methods will at
+ * least block on the resolution (success/failure) of any such pending writes.
+ *
+ * @since 1.35
+ */
+class KeyValueDependencyStore extends DependencyStore {
+	/** @var BagOStuff */
+	private $stash;
+
+	/**
+	 * @param BagOStuff $stash Storage backend
+	 */
+	public function __construct( BagOStuff $stash ) {
+		$this->stash = $stash;
+	}
+
+	public function retrieveMulti( $type, array $entities ) {
+		$entitiesByKey = [];
+		foreach ( $entities as $entity ) {
+			$entitiesByKey[$this->getStoreKey( $type, $entity )] = $entity;
+		}
+
+		$blobsByKey = $this->stash->getMulti( array_keys( $entitiesByKey ) );
+
+		$results = [];
+		foreach ( $entitiesByKey as $key => $entity ) {
+			$blob = $blobsByKey[$key] ?? null;
+			$data = is_string( $blob ) ? json_decode( $blob, true ) : null;
+			$results[$entity] = $this->newEntityDependencies(
+				$data[self::KEY_PATHS] ?? [],
+				$data[self::KEY_AS_OF] ?? null
+			);
+		}
+
+		return $results;
+	}
+
+	public function storeMulti( $type, array $dataByEntity, $ttl ) {
+		$blobsByKey = [];
+		foreach ( $dataByEntity as $entity => $data ) {
+			if ( !is_array( $data[self::KEY_PATHS] ) || !is_int( $data[self::KEY_AS_OF] ) ) {
+				throw new InvalidArgumentException( "Invalid entry for '$entity'" );
+			}
+
+			// Normalize the list by removing duplicates and sorting
+			$data[self::KEY_PATHS] = array_values( array_unique( $data[self::KEY_PATHS] ) );
+			sort( $data[self::KEY_PATHS], SORT_STRING );
+
+			$blob = json_encode( $data, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE );
+			$blobsByKey[$this->getStoreKey( $type, $entity )] = $blob;
+		}
+
+		if ( $blobsByKey ) {
+			$this->stash->setMulti( $blobsByKey, $ttl, BagOStuff::WRITE_BACKGROUND );
+		}
+	}
+
+	public function remove( $type, $entities ) {
+		$keys = [];
+		foreach ( (array)$entities as $entity ) {
+			$keys[] = $this->getStoreKey( $type, $entity );
+		}
+
+		if ( $keys ) {
+			$this->stash->deleteMulti( $keys, BagOStuff::WRITE_BACKGROUND );
+		}
+	}
+
+	public function renew( $type, $entities, $ttl ) {
+		$keys = [];
+		foreach ( (array)$entities as $entity ) {
+			$keys[] = $this->getStoreKey( $type, $entity );
+		}
+
+		if ( $keys ) {
+			$this->stash->changeTTLMulti( $keys, $ttl, BagOStuff::WRITE_BACKGROUND );
+		}
+	}
+
+	/**
+	 * @param string $type
+	 * @param string $entity
+	 * @return string
+	 */
+	private function getStoreKey( $type, $entity ) {
+		return $this->stash->makeKey( "{$type}-dependencies", $entity );
+	}
+}
diff --git a/includes/resourceloader/dependencystore/SqlModuleDependencyStore.php b/includes/resourceloader/dependencystore/SqlModuleDependencyStore.php
new file mode 100644
index 000000000000..40b41ca6f016
--- /dev/null
+++ b/includes/resourceloader/dependencystore/SqlModuleDependencyStore.php
@@ -0,0 +1,163 @@
+<?php
+/**
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ */
+
+namespace Wikimedia\DependencyStore;
+
+use InvalidArgumentException;
+use Wikimedia\Rdbms\DBError;
+use Wikimedia\Rdbms\ILoadBalancer;
+
+/**
+ * Class for tracking per-entity dependency path lists in the module_deps table
+ *
+ * This should not be used outside of ResourceLoader and ResourceLoaderModule
+ *
+ * @internal For use with ResourceLoader/ResourceLoaderModule only
+ * @since 1.35
+ */
+class SqlModuleDependencyStore extends DependencyStore {
+	/** @var ILoadBalancer */
+	private $lb;
+
+	/**
+	 * @param ILoadBalancer $lb Storage backend
+	 */
+	public function __construct( ILoadBalancer $lb ) {
+		$this->lb = $lb;
+	}
+
+	public function retrieveMulti( $type, array $entities ) {
+		try {
+			$dbr = $this->lb->getConnectionRef( DB_REPLICA );
+
+			$modulesByVariant = [];
+			foreach ( $entities as $entity ) {
+				list( $module, $variant ) = $this->getEntityNameComponents( $entity );
+				$modulesByVariant[$variant][] = $module;
+			}
+
+			$condsByVariant = [];
+			foreach ( $modulesByVariant as $variant => $modules ) {
+				$condsByVariant[] = $dbr->makeList(
+					[ 'md_module' => $modules, 'md_skin' => $variant ],
+					$dbr::LIST_AND
+				);
+			}
+
+			if ( !$condsByVariant ) {
+				return [];
+			}
+
+			$conds = $dbr->makeList( $condsByVariant, $dbr::LIST_OR );
+			$res = $dbr->select(
+				'module_deps',
+				[ 'md_module', 'md_skin', 'md_deps' ],
+				$conds,
+				__METHOD__
+			);
+
+			$pathsByEntity = [];
+			foreach ( $res as $row ) {
+				$entity = "{$row->md_module}|{$row->md_skin}";
+				$pathsByEntity[$entity] = json_decode( $row->md_deps, true );
+			}
+
+			$results = [];
+			foreach ( $entities as $entity ) {
+				$paths = $pathsByEntity[$entity] ?? [];
+				$results[$entity] = $this->newEntityDependencies( $paths, null );
+			}
+
+			return $results;
+		} catch ( DBError $e ) {
+			throw new DependencyStoreException( $e->getMessage() );
+		}
+	}
+
+	public function storeMulti( $type, array $dataByEntity, $ttl ) {
+		try {
+			$dbw = $this->lb->getConnectionRef( DB_MASTER );
+
+			$rows = [];
+			foreach ( $dataByEntity as $entity => $data ) {
+				list( $module, $variant ) = $this->getEntityNameComponents( $entity );
+				if ( !is_array( $data[self::KEY_PATHS] ) ) {
+					throw new InvalidArgumentException( "Invalid entry for '$entity'" );
+				}
+
+				// Normalize the list by removing duplicates and sortings
+				$paths = array_values( array_unique( $data[self::KEY_PATHS] ) );
+				sort( $paths, SORT_STRING );
+				$blob = json_encode( $paths, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE );
+
+				$rows[] = [
+					'md_module' => $module,
+					'md_skin' => $variant,
+					'md_deps' => $blob
+				];
+			}
+
+			if ( $rows ) {
+				$dbw->insert( 'module_deps', $rows, __METHOD__ );
+			}
+		} catch ( DBError $e ) {
+			throw new DependencyStoreException( $e->getMessage() );
+		}
+	}
+
+	public function remove( $type, $entities ) {
+		try {
+			$dbw = $this->lb->getConnectionRef( DB_MASTER );
+
+			$condsPerRow = [];
+			foreach ( (array)$entities as $entity ) {
+				list( $module, $variant ) = $this->getEntityNameComponents( $entity );
+				$condsPerRow[] = $dbw->makeList(
+					[ 'md_module' => $module, 'md_skin' => $variant ],
+					$dbw::LIST_AND
+				);
+			}
+
+			if ( $condsPerRow ) {
+				$conds = $dbw->makeList( $condsPerRow, $dbw::LIST_OR );
+				$dbw->delete( 'module_deps', $conds, __METHOD__ );
+			}
+		} catch ( DBError $e ) {
+			throw new DependencyStoreException( $e->getMessage() );
+		}
+	}
+
+	public function renew( $type, $entities, $ttl ) {
+		// no-op
+	}
+
+	/**
+	 * @param string $entity
+	 * @return string[]
+	 */
+	private function getEntityNameComponents( $entity ) {
+		$parts = explode( '|', $entity, 2 );
+		if ( count( $parts ) !== 2 ) {
+			throw new InvalidArgumentException( "Invalid module entity '$entity'" );
+		}
+
+		return $parts;
+	}
+}