Merge pull request #69 from guardian/upgrade-to-scala-v2.13-and-play-…

…v2.8

Upgrade to Scala v2.13 and Play v2.8

.gitignore

@@ -20,6 +20,7 @@ dist
 ### Scala ###
 *.class
 *.log
+.bsp
 
 # sbt specific
 .cache/

app/configuration/AppLoader.scala

@@ -0,0 +1,14 @@
+package configuration
+
+import play.api.ApplicationLoader.Context
+import play.api.{Application, ApplicationLoader, LoggerConfigurator}
+
+class AppLoader extends ApplicationLoader {
+
+  override def load(context: Context): Application = {
+    LoggerConfigurator(context.environment.classLoader).foreach {
+      _.configure(context.environment)
+    }
+    new ApplicationComponents(context).application
+  }
+}

app/configuration/ApplicationComponents.scala

@@ -0,0 +1,58 @@
+package configuration
+
+import com.madgag.scalagithub.model.User
+import com.madgag.scalagithub.{GitHub, GitHubCredentials}
+import com.softwaremill.macwire._
+import controllers.{Application, _}
+import lib.actions.Actions
+import lib.sentry.SentryApiClient
+import lib.{Bot, CheckpointSnapshoter, Delayer, Droid, PRSnapshot, PRUpdater, RepoSnapshot, RepoUpdater, ScanScheduler}
+import play.api.routing.Router
+import play.api.{ApplicationLoader, BuiltInComponentsFromContext, Logging}
+import router.Routes
+
+import java.nio.file.Path
+import scala.concurrent.Await
+import scala.concurrent.duration._
+
+class ApplicationComponents(context: ApplicationLoader.Context)
+  extends BuiltInComponentsFromContext(context) with ReasonableHttpFilters
+    with AssetsComponents with Logging {
+
+  implicit val checkpointSnapshoter: CheckpointSnapshoter = CheckpointSnapshoter
+
+  val workingDir: Path = Path.of("/tmp", "bot", "working-dir")
+
+  implicit val bot: Bot = Bot.forAccessToken(configuration.get[String]("github.botAccessToken"))
+
+  implicit val github: GitHub = bot.github
+
+  implicit val authClient: com.madgag.playgithub.auth.Client = com.madgag.playgithub.auth.Client(
+    id = configuration.get[String]("github.clientId"),
+    secret = configuration.get[String]("github.clientSecret")
+  )
+
+  val delayer: Delayer = wire[Delayer]
+  val repoSnapshotFactory: RepoSnapshot.Factory = wire[RepoSnapshot.Factory]
+
+  implicit val sentryApiClient: Option[SentryApiClient] = SentryApiClient.instanceOptFrom(configuration)
+  val repoUpdater: RepoUpdater = wire[RepoUpdater]
+  val prUpdater: PRUpdater = wire[PRUpdater]
+  val droid: Droid = wire[Droid]
+  val scanSchedulerFactory: ScanScheduler.Factory = wire[ScanScheduler.Factory]
+  val repoAcceptListService: RepoAcceptListService = wire[RepoAcceptListService]
+
+  val actions: Actions = wire[Actions]
+  val controllerAppComponents: ControllerAppComponents = wire[ControllerAppComponents]
+
+  val apiController: Api = wire[Api]
+  val appController: Application = wire[Application]
+  val authController: Auth = wire[_root_.controllers.Auth]
+
+  val router: Router = {
+    // add the prefix string in local scope for the Routes constructor
+    val prefix: String = "/"
+    wire[Routes]
+  }
+
+}

app/configuration/ReasonableHttpFilters.scala

@@ -0,0 +1,15 @@
+package configuration
+
+import play.api.mvc.EssentialFilter
+import play.filters.csrf.CSRFComponents
+import play.filters.headers.SecurityHeadersComponents
+
+/*
+This is based off the original Play class HttpFiltersComponents,
+with allowedHostsFilter removed so we can use with randomly-named
+autoscaled EC2 boxes, or whatever Heroku does.
+ */
+trait ReasonableHttpFilters extends CSRFComponents with SecurityHeadersComponents {
+
+  def httpFilters: Seq[EssentialFilter] = Seq(csrfFilter, securityHeadersFilter)
+}

app/controllers/Api.scala

@@ -16,62 +16,61 @@
 
 package controllers
 
+import com.github.blemale.scaffeine.{LoadingCache, Scaffeine}
 import com.madgag.scalagithub.model.RepoId
 import lib._
 import lib.actions.Parsers.parseGitHubHookJson
-import play.api.Logger
-import play.api.Play.current
-import play.api.cache.Cache
 import play.api.libs.json.{JsArray, JsNumber}
 import play.api.mvc._
 
-import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
 
-object Api extends Controller {
-
-  val logger = Logger(getClass)
-
-  val checkpointSnapshoter: CheckpointSnapshoter = CheckpointSnapshot(_)
+class Api(
+  scanSchedulerFactory: ScanScheduler.Factory,
+  repoAcceptListService: RepoAcceptListService,
+  delayer: Delayer,
+  cc: ControllerAppComponents
+) extends AbstractAppController(cc) {
 
   def githubHook() = Action.async(parse.json.map(parseGitHubHookJson)) { implicit request =>
     val repoId = request.body
     val githubDeliveryGuid = request.headers.get("X-GitHub-Delivery")
-    Logger.info(s"githubHook repo=${repoId.fullName} githubDeliveryGuid=$githubDeliveryGuid xRequestId=$xRequestId")
+    logger.info(s"githubHook repo=${repoId.fullName} githubDeliveryGuid=$githubDeliveryGuid xRequestId=$xRequestId")
     updateFor(repoId)
   }
 
   def updateRepo(repoId: RepoId) = Action.async { implicit request =>
-    Logger.info(s"updateRepo repo=${repoId.fullName} xRequestId=$xRequestId")
+    logger.info(s"updateRepo repo=${repoId.fullName} xRequestId=$xRequestId")
     updateFor(repoId)
   }
 
   def xRequestId(implicit request: RequestHeader): Option[String] = request.headers.get("X-Request-ID")
 
   def updateFor(repoId: RepoId): Future[Result] = {
-    Logger.debug(s"update requested for $repoId")
+    logger.debug(s"update requested for $repoId")
     for {
-      whiteList <- RepoWhitelistService.whitelist()
-      update <- updateFor(repoId, whiteList)
+      acceptList <- repoAcceptListService.acceptList()
+      update <- updateFor(repoId, acceptList)
     } yield update
   }
 
-  def updateFor(repoId: RepoId, whiteList: RepoWhitelist): Future[Result] = {
+  val repoScanSchedulerCache: LoadingCache[RepoId, ScanScheduler] = Scaffeine()
+    .recordStats()
+    .maximumSize(500)
+    .build(scanSchedulerFactory.createFor)
+
+  def updateFor(repoId: RepoId, acceptList: RepoAcceptList): Future[Result] = {
     val scanGuardF = Future { // wrapped in a future to avoid timing attacks
-      val knownRepo = whiteList.allKnownRepos(repoId)
-      Logger.debug(s"$repoId known=$knownRepo")
+      val knownRepo = acceptList.allKnownRepos(repoId)
+      logger.debug(s"$repoId known=$knownRepo")
       require(knownRepo, s"${repoId.fullName} not on known-repo whitelist")
 
-      val scanScheduler = Cache.getOrElse(repoId.fullName) {
-        val scheduler = new ScanScheduler(repoId, checkpointSnapshoter, Bot.github)
-        logger.info(s"Creating $scheduler for $repoId")
-        scheduler
-      }
-      Logger.debug(s"$repoId scanScheduler=$scanScheduler")
+      val scanScheduler = repoScanSchedulerCache.get(repoId)
+      logger.debug(s"$repoId scanScheduler=$scanScheduler")
 
       val firstScanF = scanScheduler.scan()
 
-      firstScanF.onComplete { _ => Delayer.delayTheFuture {
+      firstScanF.onComplete { _ => delayer.delayTheFuture {
         /* Do a *second* scan shortly after the first one ends, to cope with:
          * 1. Latency in GH API
          * 2. Checkpoint site stabilising on the new version after deploy
@@ -82,7 +81,7 @@ object Api extends Controller {
 
       firstScanF
     }
-    val mightBePrivate = !whiteList.publicRepos(repoId)
+    val mightBePrivate = !acceptList.publicRepos(repoId)
     if (mightBePrivate) {
       // Response must be immediate, with no private information (e.g. even acknowledging that repo exists)
       Future.successful(NoContent)

app/controllers/Application.scala

@@ -16,30 +16,35 @@
 
 package controllers
 
+import akka.actor.ActorSystem
 import com.madgag.scalagithub.model.RepoId
-import lib.actions.Actions
 import lib.{Bot, RepoSnapshot}
-import play.api.mvc._
+import play.api.Logging
 
-import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.ExecutionContext
 
-object Application extends Controller {
+class Application(
+  repoAcceptListService: RepoAcceptListService,
+  repoSnapshotFactory: RepoSnapshot.Factory,
+  sentryApiClientOpt: Option[lib.sentry.SentryApiClient],
+  cc: ControllerAppComponents
+)(implicit
+  ec: ExecutionContext,
+  bot: Bot
+) extends AbstractAppController(cc) with Logging {
 
   def index = Action { implicit req =>
     Ok(views.html.userPages.index())
   }
 
-  def zoomba(repoId: RepoId) = Actions.repoAuthenticated(repoId).async { implicit req =>
-    implicit val checkpointSnapshoter = Api.checkpointSnapshoter
+  def configDiagnostic(repoId: RepoId) = repoAuthenticated(repoId).async { implicit req =>
     for {
-      wl <- RepoWhitelistService.repoWhitelist.get()
-      repoFetchedByProut <- Bot.github.getRepo(repoId)
-      proutPresenceQuickCheck <- RepoWhitelistService.hasProutConfigFile(repoFetchedByProut)
-      repoSnapshot <- RepoSnapshot(repoFetchedByProut)
+      repoFetchedByProut <- bot.github.getRepo(repoId)
+      proutPresenceQuickCheck <- repoAcceptListService.hasProutConfigFile(repoFetchedByProut)
+      repoSnapshot <- repoSnapshotFactory.snapshot(repoFetchedByProut.repoId)
       diagnostic <- repoSnapshot.diagnostic()
     } yield {
-      val known = wl.allKnownRepos(repoId)
-      Ok(views.html.userPages.repo(proutPresenceQuickCheck, repoSnapshot, diagnostic))
+      Ok(views.html.userPages.repo(proutPresenceQuickCheck, repoSnapshot, diagnostic, sentryApiClientOpt))
     }
   }
 

app/controllers/Auth.scala

@@ -3,6 +3,9 @@ package controllers
 import com.madgag.playgithub.auth.{AuthController, Client}
 import lib.GithubAppConfig
 
-object Auth extends AuthController {
-  override val authClient: Client = GithubAppConfig.authClient
+case class Auth(
+  authClient: Client,
+  controllerComponents: ControllerAppComponents
+) extends AuthController {
+
 }

app/controllers/BaseAppController.scala

@@ -0,0 +1,38 @@
+package controllers
+
+import com.madgag.playgithub.auth.GHRequest
+import com.madgag.scalagithub.model.RepoId
+import lib.actions.Actions
+import play.api.Logging
+import play.api.http.FileMimeTypes
+import play.api.i18n.{Langs, MessagesApi}
+import play.api.mvc._
+
+import scala.concurrent.ExecutionContext
+
+case class ControllerAppComponents(
+  actions: Actions,
+  actionBuilder: DefaultActionBuilder,
+  parsers: PlayBodyParsers,
+  messagesApi: MessagesApi,
+  langs: Langs,
+  fileMimeTypes: FileMimeTypes,
+  executionContext: scala.concurrent.ExecutionContext
+) extends ControllerComponents
+
+trait BaseAppController extends BaseController with Logging {
+
+  val controllerAppComponents: ControllerAppComponents
+
+  override val controllerComponents = controllerAppComponents
+
+  implicit val ec: ExecutionContext = controllerAppComponents.executionContext // Controversial? https://www.playframework.com/documentation/2.6.x/ThreadPools
+
+  def repoAuthenticated(repoId: RepoId): ActionBuilder[GHRequest, AnyContent] =
+    controllerAppComponents.actions.repoAuthenticated(repoId)
+
+}
+
+abstract class AbstractAppController(
+  val controllerAppComponents: ControllerAppComponents
+) extends BaseAppController

app/controllers/RepoAcceptListService.scala

@@ -0,0 +1,50 @@
+package controllers
+
+import akka.actor.ActorSystem
+import akka.stream.Materializer
+import com.madgag.github.Implicits._
+import com.madgag.scalagithub.GitHub
+import com.madgag.scalagithub.model.{Repo, RepoId}
+import com.typesafe.scalalogging.LazyLogging
+import lib.ConfigFinder.ProutConfigFileName
+import lib.gitgithub._
+
+import java.util.concurrent.atomic.AtomicReference
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.Future
+import scala.concurrent.duration._
+
+case class RepoAcceptList(allKnownRepos: Set[RepoId], publicRepos: Set[RepoId])
+
+class RepoAcceptListService(
+  actorSystem: ActorSystem
+) (implicit
+  github: GitHub,
+  mat: Materializer
+) extends LazyLogging {
+
+  lazy val repoAcceptList = new AtomicReference[Future[RepoAcceptList]](getAllKnownRepos)
+
+  def acceptList(): Future[RepoAcceptList] = repoAcceptList.get()
+
+  def hasProutConfigFile(repo: Repo): Future[Boolean] = for {
+    treeT <- repo.trees2.getRecursively(s"heads/${repo.default_branch}").trying
+  } yield treeT.map(_.tree.exists(_.path.endsWith(ProutConfigFileName))).getOrElse(false)
+
+  private def getAllKnownRepos: Future[RepoAcceptList] = for { // check this to see if it always expends quota...
+    allRepos <- github.listRepos(sort="pushed", direction = "desc").take(6).all()
+    proutRepos <- Future.traverse(allRepos.filter(_.permissions.exists(_.push))) { repo =>
+      hasProutConfigFile(repo).map(hasConfig => Option.when(hasConfig)(repo))
+    }.map(_.flatten.toSet)
+  } yield RepoAcceptList(proutRepos.map(_.repoId), proutRepos.filterNot(_.`private`).map(_.repoId))
+
+
+  def start(): Unit = {
+    logger.info("Starting background repo fetch")
+    actorSystem.scheduler.scheduleWithFixedDelay(1.second, 60.seconds) { () =>
+      repoAcceptList.set(getAllKnownRepos)
+      github.checkRateLimit().foreach(status => logger.info(status.summary))
+    }
+  }
+
+}