Implemented an automated cache cleaner that keeps the FFMS2 index cache at a manageable size. Cleaning is started when video is opened with the FFMS2 video provider and runs in its own thread. Removal is based on last access time, and the FFMS2 providers now explicitly touch the index files on reading to make sure access time is updated.

By default the cache is kept below 42MB and 20 files; to change the defaults change "FFmpegSource max cache size" and "FFmpegSource max cache files" in config.dat. Either or both may be set to 0, but the cleaner will not delete the last index file in the directory, regardless of cache settings and of the file's size.

Originally committed to SVN as r2893.

aegisub/src/audio_provider_ffmpegsource.cpp

@@ -124,6 +124,9 @@ void FFmpegSourceAudioProvider::LoadAudio(Aegisub::String filename) {
 		}
 	}
 
+	// update access time of index file so it won't get cleaned away
+	wxFileName(CacheName).Touch();
+
 	// FIXME: provide a way to choose which audio track to load?
 	int TrackNumber = FFMS_GetFirstTrackOfType(Index, FFMS_TYPE_AUDIO, FFMSErrMsg, MsgSize);
 	if (TrackNumber < 0) {

aegisub/src/ffmpegsource_common.cpp

@@ -39,11 +39,20 @@
 
 ///////////
 // Headers
+#include <wx/wxprec.h>
+#include <wx/dir.h>
 #include "ffmpegsource_common.h"
 #include "md5.h"
 #include "standard_paths.h"
 #include "main.h"
 #include "frame_main.h"
+#include "options.h"
+#include <map>
+
+
+// lookit dis here static shit
+wxMutex FFmpegSourceProvider::CleaningInProgress;
+
 
 ///////////////
 // Update indexing progress
@@ -134,4 +143,124 @@ wxString FFmpegSourceProvider::GetCacheFilename(const wxString& filename)
 	return dirfn.GetShortPath() + _T("/") + fn.GetFullName();
 }
 
+/////////////////////
+// fire and forget cleaning thread (well, almost)
+bool FFmpegSourceProvider::CleanCache() {
+	wxLogDebug(_T("FFmpegSourceCacheCleaner: attempting to start thread"));
+
+	FFmpegSourceCacheCleaner *CleaningThread = new FFmpegSourceCacheCleaner(this);
+
+	if (CleaningThread->Create() != wxTHREAD_NO_ERROR) {
+		wxLogDebug(_T("FFmpegSourceCacheCleaner: thread creation failed"));
+		delete CleaningThread;
+		CleaningThread = NULL;
+		return false;
+	}
+	if (CleaningThread->Run() != wxTHREAD_NO_ERROR) {
+		wxLogDebug(_T("FFmpegSourceCacheCleaner: failed to start thread"));
+		delete CleaningThread;
+		CleaningThread = NULL;
+		return false;
+	}
+
+	wxLogDebug(_T("FFmpegSourceCacheCleaner: thread started successfully"));
+	return true;
+}
+
+
+/////////////////////
+// constructor
+FFmpegSourceCacheCleaner::FFmpegSourceCacheCleaner(FFmpegSourceProvider *par) : wxThread(wxTHREAD_DETACHED) {
+	parent = par;
+}
+
+/////////////////////
+// all actual work happens here because I was too lazy to write more functions
+wxThread::ExitCode FFmpegSourceCacheCleaner::Entry() {
+	wxMutexLocker lock(FFmpegSourceProvider::CleaningInProgress);
+	if (!lock.IsOk()) {
+		wxLogDebug(_T("FFmpegSourceCacheCleaner: cleaning already in progress, thread exiting"));
+		return (wxThread::ExitCode)1;
+	}
+
+	wxString cachedirname = StandardPaths::DecodePath(_T("?user/ffms2cache/"));
+	wxDir cachedir;
+	if (!cachedir.Open(cachedirname)) {
+		wxLogDebug(_T("FFmpegSourceCacheCleaner: couldn't open cache directory %s"), cachedirname.c_str());
+		return (wxThread::ExitCode)1;
+	}
+
+	// sleep for a bit so we (hopefully) don't thrash the disk too much while indexing is in progress
+	wxThread::This()->Sleep(2000);
+
+	// the option is in megabytes, we need bytes
+	// shift left by 20 is CLEARLY more efficient than multiplying by 1048576
+	int64_t maxsize = Options.AsInt(_T("FFmpegSource max cache size")) << 20;
+	int64_t cursize = wxDir::GetTotalSize(cachedirname).GetValue();
+	int maxfiles	= Options.AsInt(_T("FFmpegSource max cache files"));
+
+	if (!cachedir.HasFiles(_T("*.ffindex"))) {
+		wxLogDebug(_T("FFmpegSourceCacheCleaner: no index files in cache folder, exiting"));
+		return (wxThread::ExitCode)0;
+	}
+	
+	int deleted = 0;
+	int numfiles = 0;
+	std::multimap<int64_t,wxFileName> cachefiles;
+	wxString curfn_str;
+	wxFileName curfn;
+	wxDateTime curatime;
+
+	// unusually paranoid sanity check
+	// (someone might have deleted the file(s) after we did HasFiles() above; does wxDir.Open() lock the dir?)
+	if (!cachedir.GetFirst(&curfn_str, _T("*.ffindex"), wxDIR_FILES)) {
+		wxLogDebug(_T("FFmpegSourceCacheCleaner: insanity/race condition/index dir fuckery detected, exiting"));
+		return (wxThread::ExitCode)1;
+	}
+
+	numfiles++;
+	curfn = wxFileName(cachedirname, curfn_str);
+	curfn.GetTimes(&curatime, NULL, NULL);
+	// FIXME: will break when the time_t's wrap around!!1!
+	cachefiles.insert(std::pair<int64_t,wxFileName>(curatime.GetTicks(),curfn));
+
+	while (cachedir.GetNext(&curfn_str)) {
+		curfn = wxFileName(cachedirname, curfn_str);
+		curfn.GetTimes(&curatime, NULL, NULL);
+		cachefiles.insert(std::pair<int64_t,wxFileName>(curatime.GetTicks(),curfn));
+		numfiles++;
+
+		wxThread::This()->Sleep(250);
+	}
+
+	if (numfiles <= maxfiles && cursize <= maxsize) {
+		wxLogDebug(_T("FFmpegSourceCacheCleaner: cache does not need cleaning (maxsize=%d, cursize=%d; maxfiles=%d, numfiles=%d), exiting"),
+			maxsize, cursize, maxfiles, numfiles);
+		return (wxThread::ExitCode)0;
+	}
+
+	for (std::multimap<int64_t,wxFileName>::iterator i = cachefiles.begin(); i != cachefiles.end(); i++) {
+		// stop cleaning?
+		if ((cursize <= maxsize && numfiles <= maxfiles) || numfiles <= 1)
+			break;
+		
+		int64_t fsize = i->second.GetSize().GetValue();
+		if (!wxRemoveFile(i->second.GetFullPath())) {
+			wxLogDebug(_T("FFmpegSourceCacheCleaner: failed to remove file %s"),i->second.GetFullPath().c_str());
+			continue;
+		}
+		cursize -= fsize;
+		numfiles--;
+		deleted++;
+
+		wxThread::This()->Sleep(250);
+	}
+
+	wxLogDebug(_T("FFmpegSourceCacheCleaner: deleted %d files"), deleted);
+	wxLogDebug(_T("FFmpegSourceCacheCleaner: done, exiting"));
+
+	return (wxThread::ExitCode)0;
+}
+
+
 #endif WITH_FFMPEGSOURCE

aegisub/src/ffmpegsource_common.h

@@ -38,12 +38,15 @@
 
 ///////////
 // Headers
+#include <wx/wxprec.h>
+#include <wx/thread.h>
 #include "include/aegisub/aegisub.h"
 #include "../FFmpegSource2/ffms.h"
 #include "dialog_progress.h"
 
 
 class FFmpegSourceProvider {
+	friend class FFmpegSourceCacheCleaner;
 public:
 	static const int FFMSTrackMaskAll		= -1;
 	static const int FFMSTrackMaskNone		= 0;
@@ -53,9 +56,26 @@ public:
 		DialogProgress *ProgressDialog;
 	};
 
+	static wxMutex CleaningInProgress;
+	bool CleanCache();
+
 	static int FFMS_CC UpdateIndexingProgress(int State, int64_t Current, int64_t Total, void *Private);
 	FrameIndex *DoIndexing(FrameIndex *Index, wxString Filename, wxString Cachename, int Trackmask, bool IgnoreDecodeErrors);
 	wxString GetCacheFilename(const wxString& filename);
+
+	virtual FFmpegSourceProvider::~FFmpegSourceProvider() {}
 };
 
+
+class FFmpegSourceCacheCleaner : public wxThread {
+private:
+	FFmpegSourceProvider *parent;
+
+public:
+	FFmpegSourceCacheCleaner(FFmpegSourceProvider *par);
+	~FFmpegSourceCacheCleaner() {};
+	wxThread::ExitCode Entry();
+};
+
+
 #endif /* WITH_FFMPEGSOURCE */

aegisub/src/options.cpp

@@ -180,6 +180,8 @@ void OptionsManager::LoadDefaults(bool onlyDefaults,bool doOverride) {
 	SetText(_T("Subtitles Provider"),_T(DEFAULT_PROVIDER_SUBTITLE));
 	#endif
 	SetBool(_T("Video Use Pixel Shaders"),false,1700);
+	SetInt(_T("FFmpegSource max cache size"),42);
+	SetInt(_T("FFmpegSource max cache files"),20);
 
 	// Audio Options
 	SetModificationType(MOD_AUTOMATIC);

aegisub/src/video_provider_ffmpegsource.cpp

@@ -39,6 +39,7 @@
 
 ///////////
 // Headers
+#include <wx/utils.h>
 #include "include/aegisub/aegisub.h"
 #include "video_provider_ffmpegsource.h"
 #include "video_context.h"
@@ -107,6 +108,14 @@ void FFmpegSourceVideoProvider::LoadVideo(Aegisub::String filename, double fps)
 		}
 	}
 
+	// update access time of index file so it won't get cleaned away
+	wxFileName(CacheName).Touch();
+
+	// we have now read the index and may proceed with cleaning the index cache
+	if (!CleanCache()) {
+		//do something?
+	}
+
 	// set thread count
 	int Threads = Options.AsInt(_T("FFmpegSource decoding threads"));
 	if (Threads < 1)