Merge branch 'dev'

Merging in the patches for the v5 release.  THIS RELEASE INCLUDES A
CHANGE THAT REQUIRES MANUAL INTERVENTION.

It's easy, you just need to adjust your Apache configuration. But still,
a change nonetheless.

Signed-off-by: Brian Warner <brian@bdwarner.com>

.htaccess

@@ -1,5 +1,2 @@
-RewriteEngine On
-
-RewriteCond %{REQUEST_FILENAME} !-f
-
-RewriteRule ^([^\.]+)$ $1.php [NC,L]
+order deny,allow
+deny from all

utilities/facade-worker.py

@@ -50,7 +50,7 @@
 # Important: Do not modify the database number unless you've also added an
 # update clause to update_db!
 
-upstream_db = 4
+upstream_db = 5
 
 #### Database update functions ####
 
@@ -132,6 +132,49 @@ def update_db(version):
 
 		increment_db(4)
 
+	if version < 5:
+
+		add_weekly_project_cache = ("CREATE TABLE IF NOT EXISTS project_weekly_cache ("
+			"projects_id INT UNSIGNED NOT NULL,"
+			"email VARCHAR(128) NOT NULL,"
+			"affiliation VARCHAR(128),"
+			"week TINYINT UNSIGNED NOT NULL,"
+			"year SMALLINT UNSIGNED NOT NULL,"
+			"added BIGINT UNSIGNED NOT NULL,"
+			"removed BIGINT UNSIGNED NOT NULL,"
+			"whitespace BIGINT UNSIGNED NOT NULL,"
+			"files BIGINT UNSIGNED NOT NULL,"
+			"patches BIGINT UNSIGNED NOT NULL,"
+			"INDEX `projects_id,year,affiliation` (projects_id,year,affiliation),"
+			"INDEX `projects_id,year,email` (projects_id,year,email),"
+			"INDEX `projects_id,affiliation` (projects_id,affiliation),"
+			"INDEX `projects_id,email` (projects_id,email))")
+
+		cursor.execute(add_weekly_project_cache)
+		db.commit
+
+
+		add_weekly_repo_cache = ("CREATE TABLE IF NOT EXISTS repo_weekly_cache ("
+			"repos_id INT UNSIGNED NOT NULL,"
+			"email VARCHAR(128) NOT NULL,"
+			"affiliation VARCHAR(128),"
+			"week TINYINT UNSIGNED NOT NULL,"
+			"year SMALLINT UNSIGNED NOT NULL,"
+			"added BIGINT UNSIGNED NOT NULL,"
+			"removed BIGINT UNSIGNED NOT NULL,"
+			"whitespace BIGINT UNSIGNED NOT NULL,"
+			"files BIGINT UNSIGNED NOT NULL,"
+			"patches BIGINT UNSIGNED NOT NULL,"
+			"INDEX `repos_id,year,affiliation` (repos_id,year,affiliation),"
+			"INDEX `repos_id,year,email` (repos_id,year,email),"
+			"INDEX `repos_id,affiliation` (repos_id,affiliation),"
+			"INDEX `repos_id,email` (repos_id,email))")
+
+		cursor.execute(add_weekly_repo_cache)
+		db.commit
+
+		increment_db(5)
+
 	print("No further database updates.\n")
 
 def migrate_database_config():
@@ -1517,6 +1560,41 @@ def rebuild_unknown_affiliation_and_web_caches():
 
 	log_activity('Verbose','Caching projects')
 
+	cache_projects_by_week = ("INSERT INTO project_weekly_cache "
+		"SELECT r.projects_id AS projects_id, "
+		"a.%s_email AS email, "
+		"a.%s_affiliation AS affiliation, "
+		"WEEK(a.%s_date) AS week, "
+		"YEAR(a.%s_date) AS year, "
+		"SUM(a.added) AS added, "
+		"SUM(a.removed) AS removed, "
+		"SUM(a.whitespace) AS whitespace, "
+		"COUNT(DISTINCT a.filename) AS files, "
+		"COUNT(DISTINCT a.commit) AS patches "
+		"FROM analysis_data a "
+		"JOIN repos r ON r.id = a.repos_id "
+		"JOIN projects p ON p.id = r.projects_id "
+		"LEFT JOIN exclude e ON "
+			"(a.author_email = e.email "
+				"AND (e.projects_id = r.projects_id "
+					"OR e.projects_id = 0)) "
+			"OR (a.author_email LIKE CONCAT('%%',e.domain) "
+				"AND (e.projects_id = r.projects_id "
+				"OR e.projects_id = 0)) "
+		"WHERE e.email IS NULL "
+		"AND e.domain IS NULL "
+		"AND p.recache = TRUE "
+		"GROUP BY week, "
+		"year, "
+		"affiliation, "
+		"a.%s_email,"
+		"projects_id"
+		% (report_attribution,report_attribution,
+		report_date,report_date,report_attribution))
+
+	cursor.execute(cache_projects_by_week)
+	db.commit()
+
 	cache_projects_by_month = ("INSERT INTO project_monthly_cache "
 		"SELECT r.projects_id AS projects_id, "
 		"a.%s_email AS email, "
@@ -1589,6 +1667,41 @@ def rebuild_unknown_affiliation_and_web_caches():
 
 	log_activity('Verbose','Caching repos')
 
+	cache_repos_by_week = ("INSERT INTO repo_weekly_cache "
+		"SELECT a.repos_id AS repos_id, "
+		"a.%s_email AS email, "
+		"a.%s_affiliation AS affiliation, "
+		"WEEK(a.%s_date) AS week, "
+		"YEAR(a.%s_date) AS year, "
+		"SUM(a.added) AS added, "
+		"SUM(a.removed) AS removed, "
+		"SUM(a.whitespace) AS whitespace, "
+		"COUNT(DISTINCT a.filename) AS files, "
+		"COUNT(DISTINCT a.commit) AS patches "
+		"FROM analysis_data a "
+		"JOIN repos r ON r.id = a.repos_id "
+		"JOIN projects p ON p.id = r.projects_id "
+		"LEFT JOIN exclude e ON "
+			"(a.author_email = e.email "
+				"AND (e.projects_id = r.projects_id "
+					"OR e.projects_id = 0)) "
+			"OR (a.author_email LIKE CONCAT('%%',e.domain) "
+				"AND (e.projects_id = r.projects_id "
+				"OR e.projects_id = 0)) "
+		"WHERE e.email IS NULL "
+		"AND e.domain IS NULL "
+		"AND p.recache = TRUE "
+		"GROUP BY week, "
+		"year, "
+		"affiliation, "
+		"a.%s_email,"
+		"repos_id"
+		% (report_attribution,report_attribution,
+		report_date,report_date,report_attribution))
+
+	cursor.execute(cache_repos_by_week)
+	db.commit()
+
 	cache_repos_by_month = ("INSERT INTO repo_monthly_cache "
 		"SELECT a.repos_id AS repos_id, "
 		"a.%s_email AS email, "

utilities/setup.py

@@ -37,12 +37,11 @@ def create_settings(reset=0):
 # Create and populate the default settings table.
 
 	# Only increment this when you've added the support to facade-worker.py
-	database_version = 4
+	database_version = 5
 
 	# default settings
 	start_date = "2014-01-01";
-	working_dir = os.path.dirname(os.path.abspath(__file__))
-	repo_directory = os.path.join(working_dir,'../git-repos/')
+	repo_directory = os.path.join(base_dir,'git-repos/')
 
 	if reset:
 
@@ -468,6 +467,38 @@ def create_web_caches(reset=0):
 # easily add monthly LoC and patch data and get meaningful annual stats,
 # contributors can't be added across months to get to total annual number.
 
+	# Weekly caches by project
+
+	if reset:
+		clear = "DROP TABLE IF EXISTS project_weekly_cache"
+
+		# Suppress warnings about tables not existing
+
+		with warnings.catch_warnings():
+			warnings.simplefilter("ignore")
+
+			cursor.execute(clear)
+			db.commit()
+
+	create = ("CREATE TABLE IF NOT EXISTS project_weekly_cache ("
+		"projects_id INT UNSIGNED NOT NULL,"
+		"email VARCHAR(128) NOT NULL,"
+		"affiliation VARCHAR(128),"
+		"week TINYINT UNSIGNED NOT NULL,"
+		"year SMALLINT UNSIGNED NOT NULL,"
+		"added BIGINT UNSIGNED NOT NULL,"
+		"removed BIGINT UNSIGNED NOT NULL,"
+		"whitespace BIGINT UNSIGNED NOT NULL,"
+		"files BIGINT UNSIGNED NOT NULL,"
+		"patches BIGINT UNSIGNED NOT NULL,"
+		"INDEX `projects_id,year,affiliation` (projects_id,year,affiliation),"
+		"INDEX `projects_id,year,email` (projects_id,year,email),"
+		"INDEX `projects_id,affiliation` (projects_id,affiliation),"
+		"INDEX `projects_id,email` (projects_id,email))")
+
+	cursor.execute(create)
+	db.commit()
+
 	# Monthly caches by project
 
 	if reset:
@@ -529,6 +560,38 @@ def create_web_caches(reset=0):
 	cursor.execute(create)
 	db.commit()
 
+	# Weekly caches by repo
+
+	if reset:
+		clear = "DROP TABLE IF EXISTS repo_weekly_cache"
+
+		# Suppress warnings about tables not existing
+
+		with warnings.catch_warnings():
+			warnings.simplefilter("ignore")
+
+			cursor.execute(clear)
+			db.commit()
+
+	create = ("CREATE TABLE IF NOT EXISTS repo_weekly_cache ("
+		"repos_id INT UNSIGNED NOT NULL,"
+		"email VARCHAR(128) NOT NULL,"
+		"affiliation VARCHAR(128),"
+		"week TINYINT UNSIGNED NOT NULL,"
+		"year SMALLINT UNSIGNED NOT NULL,"
+		"added BIGINT UNSIGNED NOT NULL,"
+		"removed BIGINT UNSIGNED NOT NULL,"
+		"whitespace BIGINT UNSIGNED NOT NULL,"
+		"files BIGINT UNSIGNED NOT NULL,"
+		"patches BIGINT UNSIGNED NOT NULL,"
+		"INDEX `repos_id,year,affiliation` (repos_id,year,affiliation),"
+		"INDEX `repos_id,year,email` (repos_id,year,email),"
+		"INDEX `repos_id,affiliation` (repos_id,affiliation),"
+		"INDEX `repos_id,email` (repos_id,email))")
+
+	cursor.execute(create)
+	db.commit()
+
 	# Monthly caches by repo
 
 	if reset:
@@ -681,7 +744,7 @@ def create_auth(reset=0):
 
 # First make sure the database files have been setup
 
-working_dir = os.path.dirname(os.path.abspath(__file__))
+base_dir = os.path.dirname(os.path.abspath(__file__))[:-9]
 
 print ("========== Facade database setup  ==========\n\n"
 	"What do you want to do?\n"
@@ -926,8 +989,8 @@ def create_auth(reset=0):
 			'db_host_people': db_host_people}
 
 
-		creds_php_template_loc = os.path.join(working_dir,'../includes/creds.php.template')
-		creds_php_loc = os.path.join(working_dir,'../includes/creds.php')
+		creds_php_template_loc = os.path.join(base_dir,'web/includes/creds.php.template')
+		creds_php_loc = os.path.join(base_dir,'web/includes/creds.php')
 
 		creds_php_template = string.Template(open(creds_php_template_loc).read())
 
@@ -1067,8 +1130,6 @@ def create_auth(reset=0):
 
 	print ("\n========== Generating Apache2 Configs ==========\n")
 
-	facade_dir = working_dir[:-9]
-
 	print("Step 1: Create a new file in /etc/apache2/sites-available called facade.conf "
 		"with the following contents:\n\n"
 		"# Start copying here\n"
@@ -1079,7 +1140,7 @@ def create_auth(reset=0):
 		"	ErrorLog ${APACHE_LOG_DIR}/error.log\n"
 		"	CustomLog ${APACHE_LOG_DIR}/access.log combined\n"
 		"</VirtualHost>\n"
-		"# End copying here\n\n" % facade_dir)
+		"# End copying here\n\n" % os.path.join(base_dir,'web/'))
 
 	print("Step 2: Add the following lines to /etc/apache2/apache2.conf:\n\n"
 		"# Start copying here\n"
@@ -1089,7 +1150,7 @@ def create_auth(reset=0):
 		"	AllowOverride All\n"
 		"	Require all granted\n"
 		"</Directory>\n"
-		"# End copying here\n\n" % facade_dir)
+		"# End copying here\n\n" % os.path.join(base_dir,'web/'))
 
 	print("Step 3: Run this in the terminal:\n\n"
 		"  sudo a2dissite 000-default && sudo a2ensite facade && sudo a2enmod "

web/.htaccess

@@ -0,0 +1,5 @@
+RewriteEngine On
+
+RewriteCond %{REQUEST_FILENAME} !-f
+
+RewriteRule ^([^\.]+)$ $1.php [NC,L]