diff --git a/.gitignore b/.gitignore index b78eb5d..091adf1 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ dist ### Scala ### *.class *.log +.bsp # sbt specific .cache/ diff --git a/app/Global.scala b/app/Global.scala deleted file mode 100644 index 4356e26..0000000 --- a/app/Global.scala +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2014 The Guardian - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import controllers.RepoWhitelistService -import monitoring.SentryLogging -import play.api._ -import play.api.mvc._ -import play.filters.csrf._ - -object Global extends WithFilters(CSRFFilter()) with GlobalSettings { - - Logger.info("java.version="+System.getProperty("java.version")) - - override def onStart(app: Application) { - SentryLogging.init() - RepoWhitelistService.start() - } - -} \ No newline at end of file diff --git a/app/configuration/AppLoader.scala b/app/configuration/AppLoader.scala new file mode 100644 index 0000000..0fd3d76 --- /dev/null +++ b/app/configuration/AppLoader.scala @@ -0,0 +1,14 @@ +package configuration + +import play.api.ApplicationLoader.Context +import play.api.{Application, ApplicationLoader, LoggerConfigurator} + +class AppLoader extends ApplicationLoader { + + override def load(context: Context): Application = { + LoggerConfigurator(context.environment.classLoader).foreach { + _.configure(context.environment) + } + new ApplicationComponents(context).application + } +} diff --git a/app/configuration/ApplicationComponents.scala b/app/configuration/ApplicationComponents.scala new file mode 100644 index 0000000..8512181 --- /dev/null +++ b/app/configuration/ApplicationComponents.scala @@ -0,0 +1,58 @@ +package configuration + +import com.madgag.scalagithub.model.User +import com.madgag.scalagithub.{GitHub, GitHubCredentials} +import com.softwaremill.macwire._ +import controllers.{Application, _} +import lib.actions.Actions +import lib.sentry.SentryApiClient +import lib.{Bot, CheckpointSnapshoter, Delayer, Droid, PRSnapshot, PRUpdater, RepoSnapshot, RepoUpdater, ScanScheduler} +import play.api.routing.Router +import play.api.{ApplicationLoader, BuiltInComponentsFromContext, Logging} +import router.Routes + +import java.nio.file.Path +import scala.concurrent.Await +import scala.concurrent.duration._ + +class ApplicationComponents(context: ApplicationLoader.Context) + extends BuiltInComponentsFromContext(context) with ReasonableHttpFilters + with AssetsComponents with Logging { + + implicit val checkpointSnapshoter: CheckpointSnapshoter = CheckpointSnapshoter + + val workingDir: Path = Path.of("/tmp", "bot", "working-dir") + + implicit val bot: Bot = Bot.forAccessToken(configuration.get[String]("github.botAccessToken")) + + implicit val github: GitHub = bot.github + + implicit val authClient: com.madgag.playgithub.auth.Client = com.madgag.playgithub.auth.Client( + id = configuration.get[String]("github.clientId"), + secret = configuration.get[String]("github.clientSecret") + ) + + val delayer: Delayer = wire[Delayer] + val repoSnapshotFactory: RepoSnapshot.Factory = wire[RepoSnapshot.Factory] + + implicit val sentryApiClient: Option[SentryApiClient] = SentryApiClient.instanceOptFrom(configuration) + val repoUpdater: RepoUpdater = wire[RepoUpdater] + val prUpdater: PRUpdater = wire[PRUpdater] + val droid: Droid = wire[Droid] + val scanSchedulerFactory: ScanScheduler.Factory = wire[ScanScheduler.Factory] + val repoAcceptListService: RepoAcceptListService = wire[RepoAcceptListService] + + val actions: Actions = wire[Actions] + val controllerAppComponents: ControllerAppComponents = wire[ControllerAppComponents] + + val apiController: Api = wire[Api] + val appController: Application = wire[Application] + val authController: Auth = wire[_root_.controllers.Auth] + + val router: Router = { + // add the prefix string in local scope for the Routes constructor + val prefix: String = "/" + wire[Routes] + } + +} \ No newline at end of file diff --git a/app/configuration/ReasonableHttpFilters.scala b/app/configuration/ReasonableHttpFilters.scala new file mode 100644 index 0000000..c498728 --- /dev/null +++ b/app/configuration/ReasonableHttpFilters.scala @@ -0,0 +1,15 @@ +package configuration + +import play.api.mvc.EssentialFilter +import play.filters.csrf.CSRFComponents +import play.filters.headers.SecurityHeadersComponents + +/* +This is based off the original Play class HttpFiltersComponents, +with allowedHostsFilter removed so we can use with randomly-named +autoscaled EC2 boxes, or whatever Heroku does. + */ +trait ReasonableHttpFilters extends CSRFComponents with SecurityHeadersComponents { + + def httpFilters: Seq[EssentialFilter] = Seq(csrfFilter, securityHeadersFilter) +} \ No newline at end of file diff --git a/app/controllers/Api.scala b/app/controllers/Api.scala index 15718ea..f77befd 100644 --- a/app/controllers/Api.scala +++ b/app/controllers/Api.scala @@ -16,62 +16,61 @@ package controllers +import com.github.blemale.scaffeine.{LoadingCache, Scaffeine} import com.madgag.scalagithub.model.RepoId import lib._ import lib.actions.Parsers.parseGitHubHookJson -import play.api.Logger -import play.api.Play.current -import play.api.cache.Cache import play.api.libs.json.{JsArray, JsNumber} import play.api.mvc._ -import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future -object Api extends Controller { - - val logger = Logger(getClass) - - val checkpointSnapshoter: CheckpointSnapshoter = CheckpointSnapshot(_) +class Api( + scanSchedulerFactory: ScanScheduler.Factory, + repoAcceptListService: RepoAcceptListService, + delayer: Delayer, + cc: ControllerAppComponents +) extends AbstractAppController(cc) { def githubHook() = Action.async(parse.json.map(parseGitHubHookJson)) { implicit request => val repoId = request.body val githubDeliveryGuid = request.headers.get("X-GitHub-Delivery") - Logger.info(s"githubHook repo=${repoId.fullName} githubDeliveryGuid=$githubDeliveryGuid xRequestId=$xRequestId") + logger.info(s"githubHook repo=${repoId.fullName} githubDeliveryGuid=$githubDeliveryGuid xRequestId=$xRequestId") updateFor(repoId) } def updateRepo(repoId: RepoId) = Action.async { implicit request => - Logger.info(s"updateRepo repo=${repoId.fullName} xRequestId=$xRequestId") + logger.info(s"updateRepo repo=${repoId.fullName} xRequestId=$xRequestId") updateFor(repoId) } def xRequestId(implicit request: RequestHeader): Option[String] = request.headers.get("X-Request-ID") def updateFor(repoId: RepoId): Future[Result] = { - Logger.debug(s"update requested for $repoId") + logger.debug(s"update requested for $repoId") for { - whiteList <- RepoWhitelistService.whitelist() - update <- updateFor(repoId, whiteList) + acceptList <- repoAcceptListService.acceptList() + update <- updateFor(repoId, acceptList) } yield update } - def updateFor(repoId: RepoId, whiteList: RepoWhitelist): Future[Result] = { + val repoScanSchedulerCache: LoadingCache[RepoId, ScanScheduler] = Scaffeine() + .recordStats() + .maximumSize(500) + .build(scanSchedulerFactory.createFor) + + def updateFor(repoId: RepoId, acceptList: RepoAcceptList): Future[Result] = { val scanGuardF = Future { // wrapped in a future to avoid timing attacks - val knownRepo = whiteList.allKnownRepos(repoId) - Logger.debug(s"$repoId known=$knownRepo") + val knownRepo = acceptList.allKnownRepos(repoId) + logger.debug(s"$repoId known=$knownRepo") require(knownRepo, s"${repoId.fullName} not on known-repo whitelist") - val scanScheduler = Cache.getOrElse(repoId.fullName) { - val scheduler = new ScanScheduler(repoId, checkpointSnapshoter, Bot.github) - logger.info(s"Creating $scheduler for $repoId") - scheduler - } - Logger.debug(s"$repoId scanScheduler=$scanScheduler") + val scanScheduler = repoScanSchedulerCache.get(repoId) + logger.debug(s"$repoId scanScheduler=$scanScheduler") val firstScanF = scanScheduler.scan() - firstScanF.onComplete { _ => Delayer.delayTheFuture { + firstScanF.onComplete { _ => delayer.delayTheFuture { /* Do a *second* scan shortly after the first one ends, to cope with: * 1. Latency in GH API * 2. Checkpoint site stabilising on the new version after deploy @@ -82,7 +81,7 @@ object Api extends Controller { firstScanF } - val mightBePrivate = !whiteList.publicRepos(repoId) + val mightBePrivate = !acceptList.publicRepos(repoId) if (mightBePrivate) { // Response must be immediate, with no private information (e.g. even acknowledging that repo exists) Future.successful(NoContent) diff --git a/app/controllers/Application.scala b/app/controllers/Application.scala index 8a16403..3041d58 100644 --- a/app/controllers/Application.scala +++ b/app/controllers/Application.scala @@ -16,30 +16,35 @@ package controllers +import akka.actor.ActorSystem import com.madgag.scalagithub.model.RepoId -import lib.actions.Actions import lib.{Bot, RepoSnapshot} -import play.api.mvc._ +import play.api.Logging -import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.ExecutionContext -object Application extends Controller { +class Application( + repoAcceptListService: RepoAcceptListService, + repoSnapshotFactory: RepoSnapshot.Factory, + sentryApiClientOpt: Option[lib.sentry.SentryApiClient], + cc: ControllerAppComponents +)(implicit + ec: ExecutionContext, + bot: Bot +) extends AbstractAppController(cc) with Logging { def index = Action { implicit req => Ok(views.html.userPages.index()) } - def zoomba(repoId: RepoId) = Actions.repoAuthenticated(repoId).async { implicit req => - implicit val checkpointSnapshoter = Api.checkpointSnapshoter + def configDiagnostic(repoId: RepoId) = repoAuthenticated(repoId).async { implicit req => for { - wl <- RepoWhitelistService.repoWhitelist.get() - repoFetchedByProut <- Bot.github.getRepo(repoId) - proutPresenceQuickCheck <- RepoWhitelistService.hasProutConfigFile(repoFetchedByProut) - repoSnapshot <- RepoSnapshot(repoFetchedByProut) + repoFetchedByProut <- bot.github.getRepo(repoId) + proutPresenceQuickCheck <- repoAcceptListService.hasProutConfigFile(repoFetchedByProut) + repoSnapshot <- repoSnapshotFactory.snapshot(repoFetchedByProut.repoId) diagnostic <- repoSnapshot.diagnostic() } yield { - val known = wl.allKnownRepos(repoId) - Ok(views.html.userPages.repo(proutPresenceQuickCheck, repoSnapshot, diagnostic)) + Ok(views.html.userPages.repo(proutPresenceQuickCheck, repoSnapshot, diagnostic, sentryApiClientOpt)) } } diff --git a/app/controllers/Auth.scala b/app/controllers/Auth.scala index 9c455b5..e584597 100644 --- a/app/controllers/Auth.scala +++ b/app/controllers/Auth.scala @@ -3,6 +3,9 @@ package controllers import com.madgag.playgithub.auth.{AuthController, Client} import lib.GithubAppConfig -object Auth extends AuthController { - override val authClient: Client = GithubAppConfig.authClient +case class Auth( + authClient: Client, + controllerComponents: ControllerAppComponents +) extends AuthController { + } diff --git a/app/controllers/BaseAppController.scala b/app/controllers/BaseAppController.scala new file mode 100644 index 0000000..c6457d9 --- /dev/null +++ b/app/controllers/BaseAppController.scala @@ -0,0 +1,38 @@ +package controllers + +import com.madgag.playgithub.auth.GHRequest +import com.madgag.scalagithub.model.RepoId +import lib.actions.Actions +import play.api.Logging +import play.api.http.FileMimeTypes +import play.api.i18n.{Langs, MessagesApi} +import play.api.mvc._ + +import scala.concurrent.ExecutionContext + +case class ControllerAppComponents( + actions: Actions, + actionBuilder: DefaultActionBuilder, + parsers: PlayBodyParsers, + messagesApi: MessagesApi, + langs: Langs, + fileMimeTypes: FileMimeTypes, + executionContext: scala.concurrent.ExecutionContext +) extends ControllerComponents + +trait BaseAppController extends BaseController with Logging { + + val controllerAppComponents: ControllerAppComponents + + override val controllerComponents = controllerAppComponents + + implicit val ec: ExecutionContext = controllerAppComponents.executionContext // Controversial? https://www.playframework.com/documentation/2.6.x/ThreadPools + + def repoAuthenticated(repoId: RepoId): ActionBuilder[GHRequest, AnyContent] = + controllerAppComponents.actions.repoAuthenticated(repoId) + +} + +abstract class AbstractAppController( + val controllerAppComponents: ControllerAppComponents +) extends BaseAppController \ No newline at end of file diff --git a/app/controllers/RepoAcceptListService.scala b/app/controllers/RepoAcceptListService.scala new file mode 100644 index 0000000..e5ebe3d --- /dev/null +++ b/app/controllers/RepoAcceptListService.scala @@ -0,0 +1,50 @@ +package controllers + +import akka.actor.ActorSystem +import akka.stream.Materializer +import com.madgag.github.Implicits._ +import com.madgag.scalagithub.GitHub +import com.madgag.scalagithub.model.{Repo, RepoId} +import com.typesafe.scalalogging.LazyLogging +import lib.ConfigFinder.ProutConfigFileName +import lib.gitgithub._ + +import java.util.concurrent.atomic.AtomicReference +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future +import scala.concurrent.duration._ + +case class RepoAcceptList(allKnownRepos: Set[RepoId], publicRepos: Set[RepoId]) + +class RepoAcceptListService( + actorSystem: ActorSystem +) (implicit + github: GitHub, + mat: Materializer +) extends LazyLogging { + + lazy val repoAcceptList = new AtomicReference[Future[RepoAcceptList]](getAllKnownRepos) + + def acceptList(): Future[RepoAcceptList] = repoAcceptList.get() + + def hasProutConfigFile(repo: Repo): Future[Boolean] = for { + treeT <- repo.trees2.getRecursively(s"heads/${repo.default_branch}").trying + } yield treeT.map(_.tree.exists(_.path.endsWith(ProutConfigFileName))).getOrElse(false) + + private def getAllKnownRepos: Future[RepoAcceptList] = for { // check this to see if it always expends quota... + allRepos <- github.listRepos(sort="pushed", direction = "desc").take(6).all() + proutRepos <- Future.traverse(allRepos.filter(_.permissions.exists(_.push))) { repo => + hasProutConfigFile(repo).map(hasConfig => Option.when(hasConfig)(repo)) + }.map(_.flatten.toSet) + } yield RepoAcceptList(proutRepos.map(_.repoId), proutRepos.filterNot(_.`private`).map(_.repoId)) + + + def start(): Unit = { + logger.info("Starting background repo fetch") + actorSystem.scheduler.scheduleWithFixedDelay(1.second, 60.seconds) { () => + repoAcceptList.set(getAllKnownRepos) + github.checkRateLimit().foreach(status => logger.info(status.summary)) + } + } + +} diff --git a/app/controllers/RepoWhitelistService.scala b/app/controllers/RepoWhitelistService.scala deleted file mode 100644 index e44f52d..0000000 --- a/app/controllers/RepoWhitelistService.scala +++ /dev/null @@ -1,48 +0,0 @@ -package controllers - -import java.util.concurrent.atomic.AtomicReference - -import com.madgag.github.Implicits._ -import com.madgag.scalagithub.GitHub._ -import com.madgag.scalagithub.model.{Repo, RepoId} -import com.typesafe.scalalogging.LazyLogging -import lib.Bot -import lib.ConfigFinder.ProutConfigFileName -import play.api.Logger -import play.api.Play.current -import play.api.libs.concurrent.Akka - -import scala.concurrent.ExecutionContext.Implicits.global -import scala.concurrent.Future -import scala.concurrent.duration._ - -case class RepoWhitelist(allKnownRepos: Set[RepoId], publicRepos: Set[RepoId]) - -object RepoWhitelistService extends LazyLogging { - implicit val github = Bot.github - - lazy val repoWhitelist = new AtomicReference[Future[RepoWhitelist]](getAllKnownRepos) - - def whitelist(): Future[RepoWhitelist] = repoWhitelist.get() - - def hasProutConfigFile(repo: Repo): Future[Boolean] = for { - treeT <- repo.trees2.getRecursively(s"heads/${repo.default_branch}").trying - } yield treeT.map(_.tree.exists(_.path.endsWith(ProutConfigFileName))).getOrElse(false) - - private def getAllKnownRepos: Future[RepoWhitelist] = for { // check this to see if it always expends quota... - allRepos <- github.listRepos(sort="pushed", direction = "desc").takeUpTo(6) - proutRepos <- Future.traverse(allRepos.filter(_.permissions.exists(_.push))) { repo => - hasProutConfigFile(repo).map(hasConfig => if (hasConfig) Some(repo) else None) - }.map(_.flatten.toSet) - } yield RepoWhitelist(proutRepos.map(_.repoId), proutRepos.filterNot(_.`private`).map(_.repoId)) - - - def start() { - Logger.info("Starting background repo fetch") - Akka.system.scheduler.schedule(1.second, 60.seconds) { - repoWhitelist.set(getAllKnownRepos) - github.checkRateLimit().foreach(status => logger.info(status.summary)) - } - } - -} diff --git a/app/lib/Bot.scala b/app/lib/Bot.scala index fdea3ca..db72e2f 100644 --- a/app/lib/Bot.scala +++ b/app/lib/Bot.scala @@ -1,52 +1,37 @@ package lib -import com.madgag.scalagithub.model.User import com.madgag.scalagithub.{GitHub, GitHubCredentials} -import okhttp3.OkHttpClient -import play.api.Logger +import com.madgag.scalagithub.model.User +import org.eclipse.jgit.transport.CredentialsProvider +import play.api.Logging -import scala.concurrent.Await -import scala.concurrent.ExecutionContext.Implicits.global +import java.nio.file.Path +import scala.concurrent.{Await, ExecutionContext} import scala.concurrent.duration._ -import scalax.file.ImplicitConversions._ -import scalax.file.Path - -trait Bot { - - val accessToken: String - - val parentWorkDir = Path.fromString("/tmp") / "bot" / "working-dir" - - parentWorkDir.mkdirs() - - lazy val okHttpClient = { - val clientBuilder = new OkHttpClient.Builder() - - val responseCacheDir = parentWorkDir / "http-cache" - responseCacheDir.mkdirs() - if (responseCacheDir.exists) { - clientBuilder.cache(new okhttp3.Cache(responseCacheDir, 5 * 1024 * 1024)) - } else Logger.warn(s"Couldn't create HttpResponseCache dir ${responseCacheDir.path}") - clientBuilder.build() +case class Bot( + workingDir: Path, + github: GitHub, + git: CredentialsProvider, + user: User +) + +object Bot extends Logging { + def forAccessToken(accessToken: String)(implicit ec: ExecutionContext): Bot = { + val workingDir = Path.of("/tmp", "bot", "working-dir") + + val credentials: GitHubCredentials = + GitHubCredentials.forAccessKey(accessToken, workingDir).get + + val github: GitHub = new GitHub(credentials) + val user: User = Await.result(github.getUser().map(_.result), 3.seconds) + logger.info(s"Token gives GitHub user ${user.atLogin}") + + Bot( + workingDir, + github, + credentials.git, + user + ) } - - lazy val githubCredentials = GitHubCredentials.forAccessKey(accessToken, (parentWorkDir / "http-cache").toPath).get - - lazy val github = new GitHub(githubCredentials) - - lazy val user: User = { - val myself = Await.result(github.getUser(), 3 seconds) - Logger.info(s"Token '${accessToken.take(2)}...' gives GitHub user ${myself.atLogin}") - myself - } - -} - -object Bot extends Bot { - import play.api.Play.current - val config = play.api.Play.configuration.underlying - - val accessToken: String = config.getString("github.access.token") - } \ No newline at end of file diff --git a/app/lib/CheckpointSnapshot.scala b/app/lib/CheckpointSnapshot.scala index cc32194..3b05786 100644 --- a/app/lib/CheckpointSnapshot.scala +++ b/app/lib/CheckpointSnapshot.scala @@ -19,7 +19,6 @@ package lib import java.time.Instant import java.time.Instant.now import javax.net.ssl.{HostnameVerifier, SSLSession} - import com.madgag.okhttpscala._ import lib.Config.Checkpoint import lib.SSL.InsecureSocketFactory @@ -30,19 +29,22 @@ import org.eclipse.jgit.lib.{AbbreviatedObjectId, ObjectId} import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent._ import scala.util.Try +import scala.util.matching.Regex + +trait CheckpointSnapshoter { + def snapshot(checkpoint: Checkpoint): Future[Iterator[AbbreviatedObjectId]] +} -object CheckpointSnapshot { +object CheckpointSnapshoter extends CheckpointSnapshoter { val client = new OkHttpClient() - val insecureClient = new OkHttpClient.Builder() + val insecureClient: OkHttpClient = new OkHttpClient.Builder() .sslSocketFactory(InsecureSocketFactory, SSL.TrustEveryoneTrustManager) - .hostnameVerifier(new HostnameVerifier { - override def verify(hostname: String, sslSession: SSLSession): Boolean = true - }).build() + .hostnameVerifier((_, _) => true).build() - val hexRegex = """\b\p{XDigit}{40}\b""".r + val hexRegex: Regex = """\b\p{XDigit}{40}\b""".r - def apply(checkpoint: Checkpoint): Future[Iterator[AbbreviatedObjectId]] = { + def snapshot(checkpoint: Checkpoint): Future[Iterator[AbbreviatedObjectId]] = { val clientForCheckpoint = if (checkpoint.sslVerification) client else insecureClient diff --git a/app/lib/Config.scala b/app/lib/Config.scala index dde21f0..273caa0 100644 --- a/app/lib/Config.scala +++ b/app/lib/Config.scala @@ -5,7 +5,7 @@ import java.time.Instant import com.madgag.git._ import com.madgag.scalagithub.model.PullRequest import com.madgag.time.Implicits._ -import com.netaporter.uri.Uri +import io.lemonlabs.uri.Uri import lib.Config.{Checkpoint, CheckpointDetails, Sentry} import lib.labels.{Overdue, PullRequestCheckpointStatus, Seen} import org.eclipse.jgit.lib.ObjectId @@ -14,6 +14,7 @@ import org.joda.time.Period import play.api.data.validation.ValidationError import play.api.libs.json.{Json, _} import play.api.libs.functional.syntax._ +import com.madgag.scala.collection.decorators._ case class ConfigFile(checkpoints: Map[String, CheckpointDetails], sentry: Option[Sentry] = None) { @@ -31,9 +32,9 @@ object Config { def reads(json: JsValue): JsResult[T] = json match { case JsString(s) => parse(s) match { case Some(d) => JsSuccess(d) - case None => JsError(Seq(JsPath() -> Seq(ValidationError("Error parsing string")))) + case None => JsError(Seq(JsPath() -> Seq(JsonValidationError("Error parsing string")))) } - case _ => JsError(Seq(JsPath() -> Seq(ValidationError("Expected string")))) + case _ => JsError(Seq(JsPath() -> Seq(JsonValidationError("Expected string")))) } private def parse(input: String): Option[T] = @@ -112,13 +113,13 @@ object Config { val foldersWithValidConfig: Set[String] = validConfigByFolder.keySet val foldersByCheckpointName: Map[String, Seq[String]] = (for { - (folder, checkpointNames) <- validConfigByFolder.mapValues(_.checkpoints.keySet).toSeq + (folder, checkpointNames) <- validConfigByFolder.mapV(_.checkpoints.keySet).toSeq checkpointName <- checkpointNames - } yield checkpointName -> folder).groupBy(_._1).mapValues(_.map(_._2)) + } yield checkpointName -> folder).groupBy(_._1).mapV(_.map(_._2)) val checkpointsNamedInMultipleFolders: Map[String, Seq[String]] = foldersByCheckpointName.filter(_._2.size > 1) - require(checkpointsNamedInMultipleFolders.isEmpty, s"Duplicate checkpoints defined in multiple config files: ${checkpointsNamedInMultipleFolders.mapValues(_.mkString("(",", ",")"))}") + require(checkpointsNamedInMultipleFolders.isEmpty, s"Duplicate checkpoints defined in multiple config files: ${checkpointsNamedInMultipleFolders.mapV(_.mkString("(",", ",")"))}") val checkpointsByName: Map[String, Checkpoint] = validConfigByFolder.values.map(_.checkpointsByName).fold(Map.empty)(_ ++ _) } diff --git a/app/lib/ConfigFinder.scala b/app/lib/ConfigFinder.scala index e515a7e..5f6d2f8 100644 --- a/app/lib/ConfigFinder.scala +++ b/app/lib/ConfigFinder.scala @@ -5,6 +5,7 @@ import lib.Config.RepoConfig import org.eclipse.jgit.lib.ObjectId import org.eclipse.jgit.revwalk.RevCommit import org.eclipse.jgit.treewalk.TreeWalk +import com.madgag.scala.collection.decorators._ object ConfigFinder { @@ -23,7 +24,7 @@ object ConfigFinder { } def config(c: RevCommit)(implicit repoThreadLocal: ThreadLocalObjectDatabaseResources): RepoConfig = { - val checkpointsByNameByFolder = configIdMapFrom(c).mapValues(Config.readConfigFrom) + val checkpointsByNameByFolder = configIdMapFrom(c).mapV(Config.readConfigFrom) RepoConfig(checkpointsByNameByFolder) } } diff --git a/app/lib/Delayer.scala b/app/lib/Delayer.scala index c315f4d..4f623ec 100644 --- a/app/lib/Delayer.scala +++ b/app/lib/Delayer.scala @@ -1,23 +1,19 @@ package lib -import java.util.concurrent.TimeUnit - -import play.api.Play.current -import play.api.libs.concurrent.Akka +import akka.actor.ActorSystem +import java.util.concurrent.TimeUnit import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.{Future, Promise} -object Delayer { - - private implicit val system = Akka.system +class Delayer(system: ActorSystem) { def doAfterSmallDelay(f: => Unit): Unit = { system.scheduler.scheduleOnce(concurrent.duration.Duration(1, TimeUnit.SECONDS))(f) } def delayTheFuture[T](f: => Future[T]): Future[T] = { - val p = Promise[T] + val p = Promise[T]() doAfterSmallDelay(p.completeWith(f)) p.future } diff --git a/app/lib/Dogpile.scala b/app/lib/Dogpile.scala index 1de17f1..23bcd0e 100644 --- a/app/lib/Dogpile.scala +++ b/app/lib/Dogpile.scala @@ -1,7 +1,6 @@ package lib -import akka.agent.Agent - +import java.util.concurrent.atomic.AtomicReference import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.{Future, Promise} @@ -31,7 +30,7 @@ class Dogpile[R](thing: => Future[R]) { * Can run scan immediately * Must wait (future already generated or not?) - Should agent store futures? We never want the result of the *currently running* future - but we DO + Should stateRef store futures? We never want the result of the *currently running* future - but we DO want a reference to the future of the *upcoming* scan, so that we can share it among requesters. */ @@ -41,13 +40,13 @@ class Dogpile[R](thing: => Future[R]) { case class ScanRun(scanFuture: Future[R]) extends State case class ScanQueued(scanFuture: Future[R]) extends State - private val agent: Agent[State] = Agent(ScanRun(Future.failed(new IllegalStateException()))) + private val stateRef: AtomicReference[State] = new AtomicReference(ScanRun(Future.failed(new IllegalStateException()))) /** * * @return a future for a run which has been initiated at or after this call */ - def doAtLeastOneMore(): Future[R] = agent.alter { previousState => + def doAtLeastOneMore(): Future[R] = stateRef.updateAndGet { previousState => if (previousState.scanFuture.isCompleted) ScanRun(thing) else { previousState match { case ScanQueued(_) => previousState @@ -58,6 +57,6 @@ class Dogpile[R](thing: => Future[R]) { } } } - }.flatMap(_.scanFuture) + }.scanFuture } diff --git a/app/lib/Droid.scala b/app/lib/Droid.scala index 25a92de..62c2e8f 100644 --- a/app/lib/Droid.scala +++ b/app/lib/Droid.scala @@ -1,32 +1,44 @@ package lib +import akka.stream.Materializer import com.madgag.git._ -import com.madgag.scalagithub.model.Repo +import com.madgag.scalagithub.GitHub +import com.madgag.scalagithub.model.RepoId +import lib.sentry.SentryApiClient import play.api.Logger import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future -class Droid { +class Droid( + repoSnapshotFactory: RepoSnapshot.Factory, + repoUpdater: RepoUpdater, + prUpdater: PRUpdater +)(implicit + g: GitHub, + mat: Materializer, + sentryApiClientOpt: Option[SentryApiClient] +) { val logger = Logger(getClass) - def scan( - githubRepo: Repo - )(implicit checkpointSnapshoter: CheckpointSnapshoter): Future[Seq[PullRequestCheckpointsStateChangeSummary]] = { - logger.info(s"Asked to audit ${githubRepo.repoId}") - - val repoSnapshotF = RepoSnapshot(githubRepo) + def scan(repoId: RepoId): Future[Seq[PullRequestCheckpointsStateChangeSummary]] = { + logger.info(s"Asked to audit $repoId") for { - repoSnapshot <- repoSnapshotF - pullRequestUpdates <- repoSnapshot.processMergedPullRequests() + repoSnapshot <- repoSnapshotFactory.snapshot(repoId) + pullRequestUpdates <- processMergedPullRequestsOn(repoSnapshot) activeSnapshots <- repoSnapshot.activeSnapshotsF - _ <- repoSnapshot.checkForResultsOfPostDeployTesting() } yield { - logger.info(s"${githubRepo.repoId} has ${activeSnapshots.size} active snapshots : ${activeSnapshots.map(s => s.checkpoint.name -> s.commitIdTry.map(_.map(_.shortName).getOrElse("None"))).toMap}") + logger.info(s"$repoId has ${activeSnapshots.size} active snapshots : ${activeSnapshots.map(s => s.checkpoint.name -> s.commitIdTry.map(_.map(_.shortName).getOrElse("None"))).toMap}") pullRequestUpdates } } + def processMergedPullRequestsOn(repoSnapshot: RepoSnapshot): Future[Seq[PullRequestCheckpointsStateChangeSummary]] = for { + _ <- repoUpdater.attemptToCreateMissingLabels(repoSnapshot.repoLevelDetails) + summaryOpts <- Future.traverse(repoSnapshot.mergedPullRequestSnapshots)(prSnapshot => prUpdater.process(prSnapshot, repoSnapshot)) + } yield summaryOpts.flatten + + } diff --git a/app/lib/EverythingYouWantToKnowAboutACheckpoint.scala b/app/lib/EverythingYouWantToKnowAboutACheckpoint.scala index 8878b64..1aa9c9b 100644 --- a/app/lib/EverythingYouWantToKnowAboutACheckpoint.scala +++ b/app/lib/EverythingYouWantToKnowAboutACheckpoint.scala @@ -5,9 +5,9 @@ import com.madgag.scalagithub.model.PullRequest import lib.labels.{Overdue, Pending, PullRequestCheckpointStatus, Seen} import org.eclipse.jgit.lib.{ObjectId, Repository} import org.eclipse.jgit.revwalk.RevWalk -import play.api.Logger +import play.api.Logging -object EverythingYouWantToKnowAboutACheckpoint { +object EverythingYouWantToKnowAboutACheckpoint extends Logging { def apply(pr: PullRequest, snapshot: CheckpointSnapshot, gitRepo: Repository): EverythingYouWantToKnowAboutACheckpoint = { val timeBetweenMergeAndSnapshot = java.time.Duration.between(pr.merged_at.get.toInstant, snapshot.time) @@ -18,7 +18,7 @@ object EverythingYouWantToKnowAboutACheckpoint { val (prCommitsSeenOnSite, prCommitsNotSeen) = pr.availableTipCommits.partition(prCommit => w.isMergedInto(prCommit.asRevCommit, siteCommit)) if (prCommitsSeenOnSite.nonEmpty && prCommitsNotSeen.nonEmpty) { - Logger.info(s"prCommitsSeenOnSite=${prCommitsSeenOnSite.map(_.name)} prCommitsNotSeen=${prCommitsNotSeen.map(_.name)}") + logger.info(s"prCommitsSeenOnSite=${prCommitsSeenOnSite.map(_.name)} prCommitsNotSeen=${prCommitsNotSeen.map(_.name)}") } PRCommitVisibility(prCommitsSeenOnSite, prCommitsNotSeen) } diff --git a/app/lib/GitChanges.scala b/app/lib/GitChanges.scala index 12284f5..d7c118c 100644 --- a/app/lib/GitChanges.scala +++ b/app/lib/GitChanges.scala @@ -8,7 +8,7 @@ import org.eclipse.jgit.revwalk.{RevCommit, RevWalk} import org.eclipse.jgit.treewalk.TreeWalk import org.eclipse.jgit.treewalk.filter.{AndTreeFilter, PathFilterGroup, TreeFilter} -import scala.collection.convert.wrapAll._ +import scala.jdk.CollectionConverters._ object GitChanges { @@ -45,7 +45,7 @@ object GitChanges { val affectedRootPaths = rootPaths.filter(_ => mergeBase.getTree != head.getTree) val affectedSubFolderPaths = if (subFolderPaths.isEmpty) Set.empty else { - val treeFilter = AndTreeFilter.create(PathFilterGroup.createFromStrings(subFolderPaths.map(_.stripPrefix("/"))), treeDiffFilter) + val treeFilter = AndTreeFilter.create(PathFilterGroup.createFromStrings(subFolderPaths.map(_.stripPrefix("/")).asJava), treeDiffFilter) walk(mergeBase.getTree, head.getTree)(treeFilter, postOrderTraversal = true).map(_.slashPrefixedPath + "/").toSet.filter(subFolderPaths) } diff --git a/app/lib/GithubAppConfig.scala b/app/lib/GithubAppConfig.scala index 527e3dc..4beadca 100644 --- a/app/lib/GithubAppConfig.scala +++ b/app/lib/GithubAppConfig.scala @@ -1,18 +1,14 @@ package lib import com.madgag.playgithub.auth.Client +import play.api.Configuration object GithubAppConfig { - import play.api.Play.current - val config = play.api.Play.configuration - - val authClient = { - val clientId = config.getString("github.clientId").getOrElse("blah") - val clientSecret = config.getString("github.clientSecret").getOrElse("blah") - - Client(clientId, clientSecret) - } + def authClientOpt(config: Configuration): Option[Client] = for { + clientId <- config.getOptional[String]("github.clientId") + clientSecret <- config.getOptional[String]("github.clientSecret") + } yield Client(clientId, clientSecret) } diff --git a/app/lib/LabelledState.scala b/app/lib/LabelledState.scala deleted file mode 100644 index d09fa0e..0000000 --- a/app/lib/LabelledState.scala +++ /dev/null @@ -1,28 +0,0 @@ -package lib - -import com.madgag.scalagithub.GitHub._ -import com.madgag.scalagithub.model.PullRequest -import play.api.Logger - -import scala.concurrent.ExecutionContext.Implicits.global - -class LabelledState(issue: PullRequest, val applicableLabels: String => Boolean) { - - implicit val github = Bot.github - - def currentLabelsF = issue.labels.list().all().map(_.map(_.name).toSet) - - def updateLabels(newLabels: Set[String]) = for { - allOldLabels <- issue.labels.list().all() - } { - val allOldLabelsSet = allOldLabels.map(_.name).toSet - val unassociatedLabels = allOldLabelsSet.filterNot(applicableLabels) - val newLabelSet = newLabels ++ unassociatedLabels - - val labelStateChanged = newLabelSet != allOldLabelsSet - Logger.info(s"${issue.prId.slug} labelStateChanged=$labelStateChanged $newLabelSet") - if (labelStateChanged) { - issue.labels.replace(newLabelSet.toSeq) - } - } -} diff --git a/app/lib/PRSnapshot.scala b/app/lib/PRSnapshot.scala new file mode 100644 index 0000000..3914828 --- /dev/null +++ b/app/lib/PRSnapshot.scala @@ -0,0 +1,10 @@ +package lib + +import akka.stream.Materializer +import com.madgag.scalagithub.GitHub +import com.madgag.scalagithub.model.{Label, PullRequest, Repo} +import lib.gitgithub.RichSource + +import scala.concurrent.{ExecutionContext, Future} + +case class PRSnapshot(pr: PullRequest, labels: Seq[Label]) diff --git a/app/lib/PRUpdater.scala b/app/lib/PRUpdater.scala new file mode 100644 index 0000000..3224afc --- /dev/null +++ b/app/lib/PRUpdater.scala @@ -0,0 +1,139 @@ +package lib + +import akka.stream.Materializer +import com.madgag.scalagithub.GitHub +import com.madgag.scalagithub.commands.CreateComment +import com.madgag.scalagithub.model.{PullRequest, Repo} +import com.madgag.time.Implicits._ +import com.typesafe.scalalogging.LazyLogging +import lib.Config.CheckpointMessages +import lib.RepoSnapshot.WorthyOfCommentWindow +import lib.Responsibility.responsibilityAndRecencyFor +import lib.gitgithub.LabelMapping +import lib.labels.{Overdue, PullRequestCheckpointStatus, Seen} +import lib.sentry.{PRSentryRelease, SentryApiClient} + +import java.time.Instant +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future + +class PRUpdater(delayer: Delayer) extends LazyLogging { + + def process(prSnapshot: PRSnapshot, repoSnapshot: RepoSnapshot)(implicit + g: GitHub, + m: Materializer, + sentryApiClientOpt: Option[SentryApiClient] + ): Future[Option[PullRequestCheckpointsStateChangeSummary]] = { + logger.trace(s"handling ${prSnapshot.pr.prId.slug}") + for { + snapshot <- getSummaryOfCheckpointChangesGiven(prSnapshot, repoSnapshot) + } yield snapshot + } + + private def getSummaryOfCheckpointChangesGiven(prSnapshot: PRSnapshot, repoSnapshot: RepoSnapshot)(implicit + gitHub: GitHub, + sentryApiClientOpt: Option[SentryApiClient] + ): Future[Option[PullRequestCheckpointsStateChangeSummary]] = { + val pr = prSnapshot.pr + val oldLabels = prSnapshot.labels.map(_.name).toSet + val existingPersistedState: PRCheckpointState = repoSnapshot.labelToStateMapping.stateFrom(oldLabels) + if (!ignoreItemsWithExistingState(existingPersistedState)) { + for (currentSnapshot <- findCheckpointStateChange(existingPersistedState, pr, repoSnapshot)) yield { + val newPersistableState = currentSnapshot.newPersistableState + val stateChanged = newPersistableState != existingPersistedState + + logger.debug(s"handling ${pr.prId.slug} : state: existing=$existingPersistedState new=$newPersistableState stateChanged=$stateChanged") + + if (stateChanged) { + logger.info(s"#${pr.prId.slug} state-change: $existingPersistedState -> $newPersistableState") + val newLabels: Set[String] = repoSnapshot.labelToStateMapping.labelsFor(newPersistableState) + assert(oldLabels != newLabels, s"Labels should differ for differing states. labels=$oldLabels oldState=$existingPersistedState newState=$newPersistableState") + pr.labels.replace(newLabels.toSeq) + delayer.doAfterSmallDelay { + actionTaker(currentSnapshot, repoSnapshot) + } + } + Some(currentSnapshot) + } + } else Future.successful(None) + } + + + + def ignoreItemsWithExistingState(existingState: PRCheckpointState): Boolean = + existingState.hasStateForCheckpointsWhichHaveAllBeenSeen + + def findCheckpointStateChange(oldState: PRCheckpointState, pr: PullRequest, repoSnapshot: RepoSnapshot): Future[PullRequestCheckpointsStateChangeSummary] = + for (cs <- repoSnapshot.checkpointSnapshotsFor(pr, oldState)) yield { + val details = PRCheckpointDetails(pr, cs, repoSnapshot.repoLevelDetails.gitRepo) + PullRequestCheckpointsStateChangeSummary(details, oldState) + } + + def actionTaker( + checkpointsChangeSummary: PullRequestCheckpointsStateChangeSummary, + repoSnapshot: RepoSnapshot + )(implicit + g: GitHub, + sentryApiClientOpt: Option[SentryApiClient] + ): Unit = { + val pr = checkpointsChangeSummary.prCheckpointDetails.pr + val now = Instant.now() + + def sentryReleaseOpt(): Option[PRSentryRelease] = { + val sentryProjects = for { + configs <- repoSnapshot.activeConfByPullRequest.get(pr).toSeq + config <- configs + sentryConf <- config.sentry.toSeq + sentryProject <- sentryConf.projects + } yield sentryProject + + for { + mergeCommit <- pr.merge_commit_sha if sentryProjects.nonEmpty + } yield PRSentryRelease(mergeCommit, sentryProjects) + } + + val newlySeenSnapshots = checkpointsChangeSummary.changedByState.get(Seen).toSeq.flatten + + logger.info(s"action taking: ${pr.prId} newlySeenSnapshots = $newlySeenSnapshots") + + val mergeToNow = java.time.Duration.between(pr.merged_at.get.toInstant, now) + val previouslyTouchedByProut = checkpointsChangeSummary.oldState.statusByCheckpoint.nonEmpty + if (previouslyTouchedByProut || mergeToNow < WorthyOfCommentWindow) { + logger.trace(s"changedSnapshotsByState : ${checkpointsChangeSummary.changedByState}") + + def commentOn(status: PullRequestCheckpointStatus, additionalAdvice: Option[String] = None) = { + + lazy val fileFinder = repoSnapshot.repoLevelDetails.createFileFinder() + + for (changedSnapshots <- checkpointsChangeSummary.changedByState.get(status)) { + + val checkpoints = changedSnapshots.map(_.snapshot.checkpoint.nameMarkdown).mkString(", ") + + val customAdvices = for { + s <- changedSnapshots + messages <- s.snapshot.checkpoint.details.messages + path <- messages.filePathforStatus(status) + message <- fileFinder.read(path) + } yield message + val advices = if(customAdvices.nonEmpty) customAdvices else CheckpointMessages.defaults.get(status).toSet + val advice = (advices ++ additionalAdvice).mkString("\n\n") + + pr.comments2.create(CreateComment(s"${status.name} on $checkpoints (${responsibilityAndRecencyFor(pr)}) $advice")) + } + } + + for (updateReporter <- repoSnapshot.updateReporters) { + updateReporter.report(repoSnapshot, pr, checkpointsChangeSummary) + } + + val sentryDetails: Option[String] = for { + sentry <- sentryApiClientOpt + sentryRelease <- sentryReleaseOpt() + } yield sentryRelease.detailsMarkdown(sentry.org) + + commentOn(Seen, sentryDetails) + commentOn(Overdue) + } + } +} + diff --git a/app/lib/PullRequestCheckpointsStateChangeSummary.scala b/app/lib/PullRequestCheckpointsStateChangeSummary.scala index ad0917c..5c52a51 100644 --- a/app/lib/PullRequestCheckpointsStateChangeSummary.scala +++ b/app/lib/PullRequestCheckpointsStateChangeSummary.scala @@ -1,7 +1,7 @@ package lib - import com.github.nscala_time.time.Imports._ +import com.madgag.scala.collection.decorators._ import com.madgag.scalagithub.model.PullRequest import lib.Config.Checkpoint import lib.gitgithub.StateSnapshot @@ -11,35 +11,28 @@ import org.eclipse.jgit.revwalk.RevCommit case class PRCheckpointState(statusByCheckpoint: Map[String, PullRequestCheckpointStatus]) { - val checkpointsByStatus = statusByCheckpoint.groupBy(_._2).mapValues(_.keySet).withDefaultValue(Set.empty) - - def hasSeen(checkpoint: Checkpoint) = checkpointsByStatus(Seen).contains(checkpoint.name) - - def updateWith(newCheckpointStatus: Map[String, PullRequestCheckpointStatus]) = - PRCheckpointState(newCheckpointStatus ++ statusByCheckpoint.filterKeys(checkpointsByStatus(Seen))) + val isEmpty: Boolean = statusByCheckpoint.isEmpty - val states = checkpointsByStatus.keySet - - val hasStateForCheckpointsWhichHaveAllBeenSeen = states == Set(Seen) - - def all(s: PullRequestCheckpointStatus) = states.forall(_ == s) + val checkpointsByStatus: Map[PullRequestCheckpointStatus, Set[String]] = + statusByCheckpoint.groupUp(_._2)(_.keySet).withDefaultValue(Set.empty) + val states: Set[PullRequestCheckpointStatus] = checkpointsByStatus.keySet + def all(s: PullRequestCheckpointStatus): Boolean = states.forall(_ == s) def has(s: PullRequestCheckpointStatus) = states.contains(s) + val hasStateForCheckpointsWhichHaveAllBeenSeen: Boolean = states == Set(Seen) - def changeFrom(oldState: PRCheckpointState) = - (statusByCheckpoint.toSet -- oldState.statusByCheckpoint.toSet).toMap + def hasSeen(checkpoint: Checkpoint): Boolean = checkpointsByStatus(Seen).contains(checkpoint.name) - val isEmpty = statusByCheckpoint.isEmpty + def updateWith(newCheckpointStatus: Map[String, PullRequestCheckpointStatus]) = + PRCheckpointState(newCheckpointStatus ++ statusByCheckpoint.view.filterKeys(checkpointsByStatus(Seen))) + def changeFrom(oldState: PRCheckpointState): Map[String, PullRequestCheckpointStatus] = + (statusByCheckpoint.toSet -- oldState.statusByCheckpoint.toSet).toMap } case class PRCommitVisibility(seen: Set[RevCommit], unseen: Set[RevCommit]) - - - - object PRCheckpointDetails { def apply( pr: PullRequest, @@ -52,7 +45,7 @@ object PRCheckpointDetails { snapshot.checkpoint -> EverythingYouWantToKnowAboutACheckpoint(pr,snapshot,gitRepo) }).toMap - PRCheckpointDetails(pr,everythingByCheckpoint) + PRCheckpointDetails(pr, everythingByCheckpoint) } } @@ -60,11 +53,14 @@ case class PRCheckpointDetails( pr: PullRequest, everythingByCheckpoint: Map[Checkpoint, EverythingYouWantToKnowAboutACheckpoint] ) { - val checkpointStatusByName = for ((c, e) <- everythingByCheckpoint) yield c.name -> e.checkpointStatus - val everythingByCheckpointName = for ((c, e) <- everythingByCheckpoint) yield c.name -> e + val everythingByCheckpointName: Map[String, EverythingYouWantToKnowAboutACheckpoint] = + for ((c, e) <- everythingByCheckpoint) yield c.name -> e + + val checkpointStatusByName: Map[String, PullRequestCheckpointStatus] = + everythingByCheckpointName.mapV(_.checkpointStatus) val checkpointsByState: Map[PullRequestCheckpointStatus, Set[Checkpoint]] = - everythingByCheckpoint.values.groupBy(_.checkpointStatus).mapValues(_.map(_.snapshot.checkpoint).toSet) + everythingByCheckpoint.values.groupBy(_.checkpointStatus).mapV(_.map(_.snapshot.checkpoint).toSet) val soonestPendingCheckpointOverdueTime: Option[java.time.Instant] = { implicit val periodOrdering = Ordering.by[Period, Duration](_.toStandardDuration) @@ -81,9 +77,9 @@ case class PullRequestCheckpointsStateChangeSummary( val checkpointStatuses: PRCheckpointState = oldState.updateWith(prCheckpointDetails.checkpointStatusByName) - override val newPersistableState = checkpointStatuses + override val newPersistableState: PRCheckpointState = checkpointStatuses - val newlyMerged = oldState.isEmpty && !newPersistableState.isEmpty + val newlyMerged: Boolean = oldState.isEmpty && !newPersistableState.isEmpty val changed: Set[EverythingYouWantToKnowAboutACheckpoint] = checkpointStatuses.changeFrom(oldState).keySet.map(prCheckpointDetails.everythingByCheckpointName) diff --git a/app/lib/RepoSnapshot.scala b/app/lib/RepoSnapshot.scala index de5dd55..023cfb9 100644 --- a/app/lib/RepoSnapshot.scala +++ b/app/lib/RepoSnapshot.scala @@ -16,165 +16,176 @@ package lib -import java.time.{Instant, ZonedDateTime} - +import akka.stream.Materializer +import akka.stream.scaladsl.{Keep, Sink} import com.madgag.git._ import com.madgag.github.Implicits._ +import com.madgag.scala.collection.decorators._ import com.madgag.scalagithub.GitHub import com.madgag.scalagithub.GitHub._ -import com.madgag.scalagithub.commands.{CreateComment, CreateLabel} -import com.madgag.scalagithub.model.{PullRequest, Repo} +import com.madgag.scalagithub.model.{PullRequest, Repo, RepoId} import com.madgag.time.Implicits._ -import com.netaporter.uri.Uri -import com.netaporter.uri.dsl._ -import com.typesafe.scalalogging.LazyLogging -import lib.Config.{Checkpoint, CheckpointMessages} -import lib.RepoSnapshot._ -import lib.Responsibility.{createdByAndMergedByFor, responsibilityAndRecencyFor} -import lib.gitgithub.{IssueUpdater, LabelMapping} +import io.lemonlabs.uri.Url +import lib.Config.Checkpoint +import lib.gitgithub.{LabelMapping, RichSource} import lib.labels._ -import lib.librato.LibratoApiClient -import lib.librato.model.{Annotation, Link} -import lib.sentry.{PRSentryRelease, SentryApiClient} -import lib.sentry.model.CreateRelease import org.eclipse.jgit.lib.{ObjectId, Repository} import org.eclipse.jgit.revwalk.{RevCommit, RevWalk} -import play.api.Logger +import play.api.Logging +import java.time.ZonedDateTime import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent._ import scala.concurrent.duration._ import scala.util.Success -import scalax.file.ImplicitConversions._ object RepoSnapshot { - val logger = Logger(getClass) - val MaxPRsToScanPerRepo = 30 - val WorthyOfScanWindow: java.time.Duration = 14.days - val WorthyOfCommentWindow: java.time.Duration = 12.hours - val ClosedPRsMostlyRecentlyUpdated = Map( - "state" -> "closed", - "sort" -> "updated", - "direction" -> "desc" - ) + val ClosedPRsMostlyRecentlyUpdated: Map[String, String] = + Map("state" -> "closed", "sort" -> "updated", "direction" -> "desc") - def log(message: String)(implicit repo: Repo) = logger.info(s"${repo.full_name} - $message") + class Factory( + bot: Bot + )(implicit + mat: Materializer, + checkpointSnapshoter: CheckpointSnapshoter + ) { + implicit val github: GitHub = bot.github - def isMergedToMaster(pr: PullRequest)(implicit repo: Repo): Boolean = pr.merged_at.isDefined && pr.base.ref == repo.default_branch + def log(message: String)(implicit repo: Repo): Unit = logger.info(s"${repo.full_name} - $message") - def mergedPullRequestsFor(repo: Repo)(implicit g: GitHub): Future[Seq[PullRequest]] = { - val now = ZonedDateTime.now() - val timeThresholdForScan = now.minus(WorthyOfScanWindow) - def isNewEnoughToBeWorthScanning(pr: PullRequest) = pr.merged_at.exists(_.isAfter(timeThresholdForScan)) + def isMergedToMain(pr: PullRequest)(implicit repo: Repo): Boolean = + pr.merged_at.isDefined && pr.base.ref == repo.default_branch - implicit val r = repo - (for { - litePullRequests <- repo.pullRequests.list(ClosedPRsMostlyRecentlyUpdated).takeUpTo(2) - pullRequests <- Future.traverse(litePullRequests.filter(isMergedToMaster).filter(isNewEnoughToBeWorthScanning).take(MaxPRsToScanPerRepo))(pr => repo.pullRequests.get(pr.number).map(_.result)) - } yield { - log(s"PRs merged to master size=${pullRequests.size}") - pullRequests - }) andThen { case cprs => log(s"Merged Pull Requests fetched: ${cprs.map(_.map(_.number).sorted.reverse)}") } - } - - def apply(githubRepo: Repo)(implicit checkpointSnapshoter: CheckpointSnapshoter): Future[RepoSnapshot] = { - - implicit val ir: Repo = githubRepo - implicit val github = Bot.github + def snapshot(repoId: RepoId): Future[RepoSnapshot] = for { + githubRepo <- github.getRepo(repoId) + mergedPullRequests <- mergedPullRequestsFor(githubRepo) + hooksF = snapshotHooks(githubRepo) + gitRepoF = fetchLatestCopyOfGitRepo(githubRepo) + gitRepo <- gitRepoF + hooks <- hooksF + } yield RepoSnapshot( + RepoLevelDetails(githubRepo, gitRepo, hooks), + mergedPullRequests, checkpointSnapshoter + ) + + def prSnapshot(prNumber: Int)(implicit repo: Repo): Future[PRSnapshot] = for { + prResponse <- repo.pullRequests.get(prNumber) + pr = prResponse.result + labelsResponse <- pr.labels.list().all() + } yield PRSnapshot(pr, labelsResponse) + + def mergedPullRequestsFor(implicit repo: Repo): Future[Seq[PRSnapshot]] = { + val now = ZonedDateTime.now() + val timeThresholdForScan = now.minus(WorthyOfScanWindow) + + def isNewEnoughToBeWorthScanning(pr: PullRequest) = pr.merged_at.exists(_.isAfter(timeThresholdForScan)) + + (for { + litePullRequests: Seq[PullRequest] <- + repo.pullRequests.list(ClosedPRsMostlyRecentlyUpdated).take(2).all(): Future[Seq[PullRequest]] + pullRequests <- + Future.traverse(litePullRequests.filter(isMergedToMain).filter(isNewEnoughToBeWorthScanning).take(MaxPRsToScanPerRepo))(pr => prSnapshot(pr.number)) + } yield { + log(s"PRs merged to master size=${pullRequests.size}") + pullRequests + }) andThen { case cprs => log(s"Merged Pull Requests fetched: ${cprs.map(_.map(_.pr.number).sorted.reverse)}") } + } - val repoId = githubRepo.repoId + private def fetchLatestCopyOfGitRepo(implicit githubRepo: Repo): Future[Repository] = { + Future { + val repoId = githubRepo.repoId + RepoUtil.getGitRepo( + bot.workingDir.resolve(s"${repoId.owner}/${repoId.name}").toFile, + githubRepo.clone_url, + Some(bot.git)) + } andThen { case r => log(s"Git Repo ref count: ${r.map(_.getAllRefs.size)}") } + } - val hooksF: Future[Seq[Uri]] = if (githubRepo.permissions.exists(_.admin)) githubRepo.hooks.list().map(_.flatMap(_.config.get("url").map(_.uri))).all() else { + private def snapshotHooks(implicit githubRepo: Repo) = if (githubRepo.permissions.exists(_.admin)) githubRepo.hooks.list().map(_.flatMap(_.config.get("url").map(Url.parse))).all() else { log(s"No admin rights to check hooks") Future.successful(Seq.empty) } - - val gitRepoF = Future { - RepoUtil.getGitRepo( - Bot.parentWorkDir / repoId.owner / repoId.name, - githubRepo.clone_url, - Some(Bot.githubCredentials.git)) - } andThen { case r => log(s"Git Repo ref count: ${r.map(_.getAllRefs.size)}") } - - for { - mergedPullRequests <- mergedPullRequestsFor(githubRepo) - gitRepo <- gitRepoF - } yield RepoSnapshot(githubRepo, gitRepo, mergedPullRequests, hooksF, checkpointSnapshoter) - } } case class Diagnostic( - snapshots: Set[CheckpointSnapshot], - prDetails: Seq[PRCheckpointDetails] - ) { - + snapshots: Set[CheckpointSnapshot], + prDetails: Seq[PRCheckpointDetails] +) { val snapshotsByCheckpoint: Map[Checkpoint, CheckpointSnapshot] = snapshots.map(s => s.checkpoint -> s).toMap } -case class RepoSnapshot( + +case class RepoLevelDetails( repo: Repo, gitRepo: Repository, - mergedPullRequests: Seq[PullRequest], - hooksF: Future[Seq[Uri]] , - checkpointSnapshoter: CheckpointSnapshoter) { - self => - - implicit val github = Bot.github + hooks: Seq[Url] +) extends Logging { + implicit val repoThreadLocal: ThreadLocalObjectDatabaseResources = gitRepo.getObjectDatabase.threadLocalResources - implicit val repoThreadLocal = gitRepo.getObjectDatabase.threadLocalResources - - lazy val masterCommit:RevCommit = { + lazy val mainCommit:RevCommit = { val id: ObjectId = gitRepo.resolve(repo.default_branch) - Logger.info(s"Need to look at ${repo.full_name}, branch:${repo.default_branch} commit $id") + logger.info(s"Need to look at ${repo.full_name}, branch:${repo.default_branch} commit $id") assert(id != null) id.asRevCommit(new RevWalk(repoThreadLocal.reader())) } - lazy val config = ConfigFinder.config(masterCommit) + lazy val config: Config.RepoConfig = ConfigFinder.config(mainCommit) + + def createFileFinder(): FileFinder = new FileFinder(mainCommit) +} + +case class RepoSnapshot( + repoLevelDetails: RepoLevelDetails, + mergedPullRequestSnapshots: Seq[PRSnapshot], + checkpointSnapshoter: CheckpointSnapshoter +) extends Logging { + + val repo: Repo = repoLevelDetails.repo + val config: Config.RepoConfig = repoLevelDetails.config + + val mergedPRs: Seq[PullRequest] = mergedPullRequestSnapshots.map(_.pr) + + val updateReporters: Seq[UpdateReporter] = Seq.empty lazy val affectedFoldersByPullRequest: Map[PullRequest, Set[String]] = { - implicit val revWalk = new RevWalk(repoThreadLocal.reader()) - println(s"getting affectedFoldersByPullRequest on ${gitRepo.getDirectory.getAbsolutePath}") + implicit val revWalk = new RevWalk(repoLevelDetails.repoThreadLocal.reader()) (for { - pr <- mergedPullRequests - } yield pr -> GitChanges.affects(pr, config.foldersWithValidConfig)).toMap + pr <- mergedPRs + } yield pr -> GitChanges.affects(pr, repoLevelDetails.config.foldersWithValidConfig)).toMap } - - lazy val pullRequestsByAffectedFolder : Map[String, Set[PullRequest]] = config.foldersWithValidConfig.map { - folder => folder -> mergedPullRequests.filter(pr => affectedFoldersByPullRequest(pr).contains(folder)).toSet + lazy val pullRequestsByAffectedFolder : Map[String, Set[PullRequest]] = repoLevelDetails.config.foldersWithValidConfig.map { + folder => folder -> mergedPRs.filter(pr => affectedFoldersByPullRequest(pr).contains(folder)).toSet }.toMap - Logger.info(s"${repo.full_name} pullRequestsByAffectedFolder : ${pullRequestsByAffectedFolder.mapValues(_.map(_.number))}") + logger.info(s"${repoLevelDetails.repo.full_name} pullRequestsByAffectedFolder : ${pullRequestsByAffectedFolder.mapV(_.map(_.number))}") - lazy val activeConfByPullRequest: Map[PullRequest, Set[ConfigFile]] = affectedFoldersByPullRequest.mapValues { - _.map(config.validConfigByFolder(_)) + lazy val activeConfByPullRequest: Map[PullRequest, Set[ConfigFile]] = affectedFoldersByPullRequest.mapV { + _.map(repoLevelDetails.config.validConfigByFolder(_)) } - lazy val activeCheckpointsByPullRequest: Map[PullRequest, Set[Checkpoint]] = activeConfByPullRequest.mapValues { + lazy val activeCheckpointsByPullRequest: Map[PullRequest, Set[Checkpoint]] = activeConfByPullRequest.mapV { _.flatMap(_.checkpointSet) } - val allAvailableCheckpoints: Set[Checkpoint] = config.checkpointsByName.values.toSet + val allAvailableCheckpoints: Set[Checkpoint] = repoLevelDetails.config.checkpointsByName.values.toSet val allPossibleCheckpointPRLabels: Set[String] = for { prLabel <- PullRequestLabel.all checkpoint <- allAvailableCheckpoints } yield prLabel.labelFor(checkpoint.name) - def diagnostic(): Future[Diagnostic] = { - for { - snapshots <- snapshotOfAllAvailableCheckpoints() - } yield { - Diagnostic(snapshots, mergedPullRequests.map(pr => PRCheckpointDetails(pr, snapshots.filter(s => activeCheckpointsByPullRequest(pr).contains(s.checkpoint)), gitRepo))) - } - } + def diagnostic(): Future[Diagnostic] = for { + snapshots <- snapshotOfAllAvailableCheckpoints() + } yield Diagnostic(snapshots, mergedPRs.map { pr => + PRCheckpointDetails(pr, snapshots.filter(s => activeCheckpointsByPullRequest(pr).contains(s.checkpoint)), repoLevelDetails.gitRepo) + }) def snapshotOfAllAvailableCheckpoints(): Future[Set[CheckpointSnapshot]] = Future.sequence(allAvailableCheckpoints.map(takeCheckpointSnapshot)) @@ -184,183 +195,31 @@ case class RepoSnapshot( lazy val snapshotsOfActiveCheckpointsF: Map[Checkpoint, Future[CheckpointSnapshot]] = activeCheckpoints.map { c => c -> takeCheckpointSnapshot(c) }.toMap - def takeCheckpointSnapshot(checkpoint: Checkpoint): Future[CheckpointSnapshot] = { - for (possibleIdsTry <- checkpointSnapshoter(checkpoint).trying) yield { - val objectIdTry = for (possibleIds <- possibleIdsTry) yield { - possibleIds.map(repoThreadLocal.reader().resolveExistingUniqueId).collectFirst { - case Success(objectId) => objectId - } + def takeCheckpointSnapshot(checkpoint: Checkpoint): Future[CheckpointSnapshot] = for ( + possibleIdsTry <- checkpointSnapshoter.snapshot(checkpoint).trying + ) yield { + val objectIdTry = for (possibleIds <- possibleIdsTry) yield { + possibleIds.map(repoLevelDetails.repoThreadLocal.reader().resolveExistingUniqueId).collectFirst { + case Success(objectId) => objectId } - CheckpointSnapshot(checkpoint, objectIdTry) } + CheckpointSnapshot(checkpoint, objectIdTry) } - lazy val activeSnapshotsF = Future.sequence(activeCheckpoints.map(snapshotsOfActiveCheckpointsF)) + lazy val activeSnapshotsF: Future[Set[CheckpointSnapshot]] = + Future.sequence(activeCheckpoints.map(snapshotsOfActiveCheckpointsF)) def checkpointSnapshotsFor(pr: PullRequest, oldState: PRCheckpointState): Future[Set[CheckpointSnapshot]] = Future.sequence(activeCheckpointsByPullRequest(pr).filter(!oldState.hasSeen(_)).map(snapshotsOfActiveCheckpointsF)) - val issueUpdater = new IssueUpdater[PullRequest, PRCheckpointState, PullRequestCheckpointsStateChangeSummary] with LazyLogging { - val repo = self.repo - - val repoSnapshot: RepoSnapshot = self + val labelToStateMapping: LabelMapping[PRCheckpointState] = new LabelMapping[PRCheckpointState] { + def labelsFor(s: PRCheckpointState): Set[String] = s.statusByCheckpoint.map { + case (checkpointName, cs) => cs.labelFor(checkpointName) + }.toSet - val labelToStateMapping = new LabelMapping[PRCheckpointState] { - def labelsFor(s: PRCheckpointState): Set[String] = s.statusByCheckpoint.map { - case (checkpointName, cs) => cs.labelFor(checkpointName) - }.toSet - - def stateFrom(labels: Set[String]): PRCheckpointState = PRCheckpointState(activeCheckpoints.flatMap { checkpoint => - PullRequestCheckpointStatus.fromLabels(labels, checkpoint).map(checkpoint.name -> _) - }.toMap) - } - - def ignoreItemsWithExistingState(existingState: PRCheckpointState): Boolean = - existingState.hasStateForCheckpointsWhichHaveAllBeenSeen - - def snapshot(oldState: PRCheckpointState, pr: PullRequest) = - for (cs <- checkpointSnapshotsFor(pr, oldState)) yield { - val details = PRCheckpointDetails(pr, cs, gitRepo) - PullRequestCheckpointsStateChangeSummary(details, oldState) - } - - override def actionTaker(snapshot: PullRequestCheckpointsStateChangeSummary) { - val pr = snapshot.prCheckpointDetails.pr - val now = Instant.now() - - def sentryReleaseOpt(): Option[PRSentryRelease] = { - val sentryProjects = for { - configs <- activeConfByPullRequest.get(pr).toSeq - config <- configs - sentryConf <- config.sentry.toSeq - sentryProject <- sentryConf.projects - } yield sentryProject - - for { - mergeCommit <- pr.merge_commit_sha if sentryProjects.nonEmpty - } yield PRSentryRelease(mergeCommit, sentryProjects) - } - - if (snapshot.newlyMerged) { - activeCheckpointsByPullRequest - logger.info(s"action taking: ${pr.prId} is newly merged") - - for { - sentry <- SentryApiClient.instanceOpt.toSeq - sentryRelease <- sentryReleaseOpt() - } { - val ref = lib.sentry.model.Ref( - repo.repoId, - sentryRelease.mergeCommit, - sentryRelease.mergeCommit.asRevCommit(new RevWalk(repoThreadLocal.reader())).getParents.headOption) - logger.info(s"${pr.prId.slug} : ref=$ref") - - sentry.createRelease(CreateRelease( - sentryRelease.version, - Some(sentryRelease.version), - Some(pr.html_url), - sentryRelease.projects, - refs=Seq(ref) - )) - } - } - - - val newlySeenSnapshots = snapshot.changedByState.get(Seen).toSeq.flatten - - logger.info(s"action taking: ${pr.prId} newlySeenSnapshots = $newlySeenSnapshots") - - for { - newlySeenSnapshot <- newlySeenSnapshots - } { - val checkpoint = newlySeenSnapshot.snapshot.checkpoint - for { librato <- LibratoApiClient.instanceOpt } { - librato.createAnnotation(s"${pr.baseRepo.name}.prout", Annotation( - title = s"PR #${pr.number} : '${pr.title}' deployed", - description = Some(createdByAndMergedByFor(pr).capitalize), - start_time = pr.merged_at.map(_.toInstant), - end_time = Some(now), - source = Some(checkpoint.name), - links = Seq(Link( - rel = "github", - label = Some(s"PR #${pr.number}"), - href = Uri.parse(pr.html_url) - )) - )) - } - } - - val mergeToNow = java.time.Duration.between(pr.merged_at.get.toInstant, now) - val previouslyTouchedByProut = snapshot.oldState.statusByCheckpoint.nonEmpty - if (previouslyTouchedByProut || mergeToNow < WorthyOfCommentWindow) { - logger.trace(s"changedSnapshotsByState : ${snapshot.changedByState}") - - def commentOn(status: PullRequestCheckpointStatus, additionalAdvice: Option[String] = None) = { - - lazy val fileFinder = new FileFinder(masterCommit) - - for (changedSnapshots <- snapshot.changedByState.get(status)) { - - val checkpoints = changedSnapshots.map(_.snapshot.checkpoint.nameMarkdown).mkString(", ") - - val customAdvices = for { - s <- changedSnapshots - messages <- s.snapshot.checkpoint.details.messages - path <- messages.filePathforStatus(status) - message <- fileFinder.read(path) - } yield message - val advices = if(customAdvices.nonEmpty) customAdvices else CheckpointMessages.defaults.get(status).toSet - val advice = (advices ++ additionalAdvice).mkString("\n\n") - - pr.comments2.create(CreateComment(s"${status.name} on $checkpoints (${responsibilityAndRecencyFor(pr)}) $advice")) - } - } - - for (hooks <- hooksF) { - slack.DeployReporter.report(snapshot, hooks) - } - - val sentryDetails: Option[String] = for { - sentry <- SentryApiClient.instanceOpt - sentryRelease <- sentryReleaseOpt() - } yield sentryRelease.detailsMarkdown(sentry.org) - - commentOn(Seen, sentryDetails) - commentOn(Overdue) - } - } - - - } - - def processMergedPullRequests(): Future[Seq[PullRequestCheckpointsStateChangeSummary]] = for { - _ <- attemptToCreateMissingLabels() - summaryOpts <- Future.traverse(mergedPullRequests)(issueUpdater.process) - } yield summaryOpts.flatten - - def missingLabelsGiven(existingLabelNames: Set[String]): Set[CreateLabel] = for { - prcs <- (PullRequestCheckpointStatus.all ++ CheckpointTestStatus.all) - checkpointName <- config.checkpointsByName.keySet - label = prcs.labelFor(checkpointName) - if !existingLabelNames(label) - } yield CreateLabel(label, prcs.defaultColour) - - def attemptToCreateMissingLabels(): Future[_] = { - for { - existingLabels <- repo.labels.list().all() - createdLabels <- Future.traverse(missingLabelsGiven(existingLabels.map(_.name).toSet)) { - missingLabel => repo.labels.create(missingLabel) - } - } yield createdLabels - }.trying - - def prByMasterCommitOpt = mergedPullRequests.find(_.merge_commit_sha.contains(masterCommit.toObjectId)) - - def checkForResultsOfPostDeployTesting() = { - // TiP currently works only when there is a single checkpoint with after seen instructions - activeSnapshotsF.map { activeSnapshots => - () - } + def stateFrom(labels: Set[String]): PRCheckpointState = PRCheckpointState(activeCheckpoints.flatMap { checkpoint => + PullRequestCheckpointStatus.fromLabels(labels, checkpoint).map(checkpoint.name -> _) + }.toMap) } } diff --git a/app/lib/RepoUpdater.scala b/app/lib/RepoUpdater.scala new file mode 100644 index 0000000..5a43486 --- /dev/null +++ b/app/lib/RepoUpdater.scala @@ -0,0 +1,35 @@ +package lib + +import akka.stream.Materializer +import akka.stream.scaladsl.{Keep, Sink} +import com.madgag.github.Implicits.RichFuture +import com.madgag.scalagithub.GitHub +import com.madgag.scalagithub.commands.CreateLabel +import com.madgag.scalagithub.model.{Label, Repo} +import lib.gitgithub.RichSource +import lib.labels.{CheckpointTestStatus, PullRequestCheckpointStatus} + +import scala.concurrent.{ExecutionContext, Future} + +class RepoUpdater(implicit + g: GitHub, + m: Materializer, + ec: ExecutionContext +) { + + def attemptToCreateMissingLabels(repoLevelDetails: RepoLevelDetails): Future[_] = { + for { + existingLabels <- repoLevelDetails.repo.labels.list().all() + createdLabels <- Future.traverse(missingLabelsGiven(repoLevelDetails, existingLabels.map(_.name).toSet)) { + missingLabel => repoLevelDetails.repo.labels.create(missingLabel) + } + } yield createdLabels + }.trying + + def missingLabelsGiven(repoLevelDetails: RepoLevelDetails, existingLabelNames: Set[String]): Set[CreateLabel] = for { + prcs <- PullRequestCheckpointStatus.all ++ CheckpointTestStatus.all + checkpointName <- repoLevelDetails.config.checkpointsByName.keySet + label = prcs.labelFor(checkpointName) + if !existingLabelNames(label) + } yield CreateLabel(label, prcs.defaultColour) +} diff --git a/app/lib/RepoUtil.scala b/app/lib/RepoUtil.scala index e962eb5..4d1e988 100644 --- a/app/lib/RepoUtil.scala +++ b/app/lib/RepoUtil.scala @@ -16,20 +16,20 @@ package lib -import java.io.File - import com.madgag.git._ import org.eclipse.jgit.api.{Git, GitCommand, TransportCommand} import org.eclipse.jgit.lib.Constants.DEFAULT_REMOTE_NAME import org.eclipse.jgit.lib.Repository import org.eclipse.jgit.storage.file.FileRepositoryBuilder import org.eclipse.jgit.transport.{CredentialsProvider, RemoteConfig} -import play.api.Logger +import play.api.Logging + +import java.io.File +import java.nio.file.Files +import scala.jdk.CollectionConverters._ -import scala.collection.convert.wrapAsScala._ -import scalax.file.ImplicitConversions._ -object RepoUtil { +object RepoUtil extends Logging { def getGitRepo(dataDirectory: File, uri: String, credentials: Option[CredentialsProvider] = None): Repository = { @@ -42,23 +42,22 @@ object RepoUtil { def getUpToDateRepo(): Repository = { dataDirectory.mkdirs() - val gitdir = dataDirectory / "repo.git" + val gitdir = new File(dataDirectory,"repo.git") - if (gitdir.exists) { - val gitDirChildren = gitdir.children().toList.map(_.name).sorted - assert(gitDirChildren.nonEmpty, s"No child files found in ${gitdir.getAbsolutePath}") - Logger.info(s"Updating Git repo with fetch... $uri") + if (gitdir.exists()) { + assert(gitdir.list().nonEmpty, s"No child files found in ${gitdir.getAbsolutePath}") + logger.info(s"Updating Git repo with fetch... $uri") val repo = FileRepositoryBuilder.create(gitdir) val remoteConfig = new RemoteConfig(repo.getConfig, DEFAULT_REMOTE_NAME) val remoteUris = remoteConfig.getURIs - val remoteUri = remoteUris.headOption.getOrElse(throw new IllegalStateException(s"No remote configured for $uri")).toString + val remoteUri = remoteUris.asScala.headOption.getOrElse(throw new IllegalStateException(s"No remote configured for $uri")).toString assert(remoteUri == uri, s"Wrong uri - expected $uri, got $remoteUri") invoke(repo.git.fetch()) repo } else { - gitdir.doCreateParents() - Logger.info(s"Cloning new Git repo... $uri") + Files.createDirectories(gitdir.toPath.getParent) + logger.info(s"Cloning new Git repo... $uri") invoke(Git.cloneRepository().setBare(true).setDirectory(gitdir).setURI(uri)).getRepository } } diff --git a/app/lib/Responsibility.scala b/app/lib/Responsibility.scala index eeb4ed8..9e99aac 100644 --- a/app/lib/Responsibility.scala +++ b/app/lib/Responsibility.scala @@ -1,15 +1,14 @@ package lib import java.time.Instant.now - import com.madgag.scalagithub.model.PullRequest import com.madgag.time.Implicits._ import com.typesafe.scalalogging.LazyLogging -import org.joda.time.format.PeriodFormat +import org.joda.time.format.{PeriodFormat, PeriodFormatter} object Responsibility extends LazyLogging { - val pf = PeriodFormat.getDefault + val pf: PeriodFormatter = PeriodFormat.getDefault def responsibilityAndRecencyFor(pr: PullRequest): String = { val mergeToNow = java.time.Duration.between(pr.merged_at.get.toInstant, now) @@ -20,12 +19,9 @@ object Responsibility extends LazyLogging { } def createdByAndMergedByFor(pr: PullRequest): String = { - val mergedByOpt = pr.merged_by val mergedByText = s"merged by ${mergedByOpt.get.atLogin}" - if (pr.user.id == mergedByOpt.get.id) mergedByText else { - s"created by ${pr.user.atLogin} and $mergedByText" - } + if (pr.user.id == mergedByOpt.get.id) mergedByText else s"created by ${pr.user.atLogin} and $mergedByText" } } diff --git a/app/lib/SSL.scala b/app/lib/SSL.scala index 3c222b0..5888d3f 100644 --- a/app/lib/SSL.scala +++ b/app/lib/SSL.scala @@ -1,23 +1,23 @@ package lib +import play.api.Logging + import java.security.cert.X509Certificate import javax.net.ssl._ -import play.api.Logger - object SSL { - val InsecureSocketFactory = { - val sslcontext = SSLContext.getInstance("TLS") - sslcontext.init(null, Array(TrustEveryoneTrustManager), null) - sslcontext.getSocketFactory + val InsecureSocketFactory: SSLSocketFactory = { + val sslContext = SSLContext.getInstance("TLS") + sslContext.init(null, Array(TrustEveryoneTrustManager), null) + sslContext.getSocketFactory } - object TrustEveryoneTrustManager extends X509TrustManager { - def checkClientTrusted(chain: Array[X509Certificate], authType: String) {} + object TrustEveryoneTrustManager extends X509TrustManager with Logging { + def checkClientTrusted(chain: Array[X509Certificate], authType: String): Unit = {} - def checkServerTrusted(chain: Array[X509Certificate], authType: String) { - Logger.warn("Skipping SSL server chain verification") + def checkServerTrusted(chain: Array[X509Certificate], authType: String): Unit = { + logger.warn("Skipping SSL server chain verification") } val getAcceptedIssuers = new Array[X509Certificate](0) diff --git a/app/lib/ScanScheduler.scala b/app/lib/ScanScheduler.scala index aeb66af..c338a0b 100644 --- a/app/lib/ScanScheduler.scala +++ b/app/lib/ScanScheduler.scala @@ -1,43 +1,62 @@ package lib +import akka.actor.ActorSystem + import java.time.Instant import java.time.Instant.now import java.time.temporal.ChronoUnit.MINUTES - -import akka.agent.Agent import com.madgag.github.Implicits._ import com.madgag.scalagithub.GitHub import com.madgag.scalagithub.model.RepoId import com.madgag.time.Implicits._ import lib.labels.Seen -import play.api.Logger -import play.api.Play.current +import play.api.Logging import play.api.libs.concurrent.Akka +import java.util.concurrent.atomic.AtomicReference import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future import scala.util.{Failure, Success} -class ScanScheduler(repoId: RepoId, - checkpointSnapshoter: CheckpointSnapshoter, - conn: GitHub) { selfScanScheduler => +object ScanScheduler { + class Factory( + droid: Droid, + conn: GitHub, + actorSystem: ActorSystem, + delayer: Delayer + ) extends Logging { + def createFor(repoId: RepoId): ScanScheduler = { + logger.info(s"Creating scheduler for $repoId") + new ScanScheduler( + repoId, + droid, + actorSystem, + delayer + ) + } + } +} - val droid = new Droid +class ScanScheduler( + repoId: RepoId, + droid: Droid, + actorSystem: ActorSystem, + delayer: Delayer +) extends Logging { selfScanScheduler => - val earliestFollowUpScanTime = Agent(now) + val earliestFollowUpScanTime: AtomicReference[Instant] = new AtomicReference(Instant.now()) - private val dogpile = new Dogpile(Delayer.delayTheFuture { - Logger.debug(s"In the dogpile for $repoId...") + private val dogpile = new Dogpile(delayer.delayTheFuture { + logger.debug(s"In the dogpile for $repoId...") + val summariesF = droid.scan(repoId) for { - repo <- conn.getRepo(repoId) - summariesF = droid.scan(repo)(checkpointSnapshoter) summariesTry <- summariesF.trying } yield { summariesTry match { case Failure(e) => - Logger.error(s"Scanning $repoId failed", e) + logger.error(s"Scanning $repoId failed", e) case Success(summaries) => - Logger.info(s"$selfScanScheduler : ${summaries.size} summaries for ${repoId.fullName}:\n${summaries.map(s => s"#${s.prCheckpointDetails.pr.prId.slug} changed=${s.changed.map(_.snapshot.checkpoint.name)}").mkString("\n")}") + logger.info(s"$selfScanScheduler : ${summaries.size} summaries for ${repoId.fullName}:\n${summaries.map(s => s"#${s.prCheckpointDetails.pr.prId.slug} changed=${s.changed.map(_.snapshot.checkpoint.name)}").mkString("\n")}") val scanTimeForUnseenOpt = summaries.find(!_.checkpointStatuses.all(Seen)).map(_ => now.plus(1L, MINUTES)) @@ -49,10 +68,10 @@ class ScanScheduler(repoId: RepoId, if (candidateFollowUpScanTimes.nonEmpty) { val earliestCandidateScanTime: Instant = candidateFollowUpScanTimes.min - earliestFollowUpScanTime.send { + earliestFollowUpScanTime.updateAndGet { oldFollowupTime => if (now.isAfter(oldFollowupTime) || earliestCandidateScanTime.isBefore(oldFollowupTime)) { - Akka.system.scheduler.scheduleOnce(java.time.Duration.between(now, earliestCandidateScanTime)) { + actorSystem.scheduler.scheduleOnce(java.time.Duration.between(now, earliestCandidateScanTime)) { scan() } earliestCandidateScanTime diff --git a/app/lib/UpdateReporter.scala b/app/lib/UpdateReporter.scala new file mode 100644 index 0000000..d7e4c79 --- /dev/null +++ b/app/lib/UpdateReporter.scala @@ -0,0 +1,11 @@ +package lib + +import com.madgag.scalagithub.model.PullRequest + +trait UpdateReporter { + def report( + repoSnapshot: RepoSnapshot, + pr: PullRequest, + checkpointsStateChangeSummary: PullRequestCheckpointsStateChangeSummary + ): Unit +} diff --git a/app/lib/actions/Actions.scala b/app/lib/actions/Actions.scala index c28d82e..e716f06 100644 --- a/app/lib/actions/Actions.scala +++ b/app/lib/actions/Actions.scala @@ -2,27 +2,33 @@ package lib.actions import com.madgag.github.Implicits._ import com.madgag.playgithub.auth.AuthenticatedSessions.AccessToken -import com.madgag.playgithub.auth.{Client, GHRequest} +import com.madgag.playgithub.auth.AuthenticatedSessions.AccessToken.Provider +import com.madgag.playgithub.auth.GHRequest import com.madgag.scalagithub.model.RepoId -import controllers.Application._ -import controllers.{Auth, routes} +import controllers.routes import lib._ -import play.api.mvc.{ActionFilter, Result} - -import scala.concurrent.ExecutionContext.Implicits.global -import scala.concurrent.Future -import scalax.file.ImplicitConversions._ - -object Actions { +import play.api.mvc.Results.Redirect +import play.api.mvc._ + +import scala.concurrent.{ExecutionContext, Future} + +class Actions( + bot: Bot, + bodyParser: BodyParser[AnyContent] +)(implicit + authClient: com.madgag.playgithub.auth.Client, + ec: ExecutionContext +) { private val authScopes = Seq("repo") - implicit val authClient: Client = Auth.authClient + implicit val provider: Provider = AccessToken.FromSession - implicit val provider = AccessToken.FromSession + val GitHubAuthenticatedAction: ActionBuilder[GHRequest, AnyContent] = + com.madgag.playgithub.auth.Actions.gitHubAction(authScopes, bot.workingDir, bodyParser) - val GitHubAuthenticatedAction = com.madgag.playgithub.auth.Actions.gitHubAction(authScopes, Bot.parentWorkDir.toPath) + def repoAccessFilter(repoId: RepoId): ActionFilter[GHRequest] = new ActionFilter[GHRequest] { + def executionContext = ec - def repoAccessFilter(repoId: RepoId) = new ActionFilter[GHRequest] { override protected def filter[A](req: GHRequest[A]): Future[Option[Result]] = { for { user <- req.userF diff --git a/app/lib/actions/Functions.scala b/app/lib/actions/Functions.scala index 4b4dd09..b73e568 100644 --- a/app/lib/actions/Functions.scala +++ b/app/lib/actions/Functions.scala @@ -1,14 +1,9 @@ package lib.actions -import javax.crypto.Mac -import javax.crypto.spec.SecretKeySpec - -import org.apache.commons.codec.binary.Base64.decodeBase64 import play.api.libs.Codecs -import play.api.mvc.Results.Unauthorized -import play.api.mvc.Security.AuthenticatedBuilder -import play.api.mvc._ +import javax.crypto.Mac +import javax.crypto.spec.SecretKeySpec object Functions { @@ -17,31 +12,4 @@ object Functions { mac.init(new SecretKeySpec(key, "HmacSHA1")) Codecs.toHexString(mac.doFinal(message)) } -} - -object BasicAuth { - - case class Credentials(username: String, password: String) - - private val requestBasicAuth = Unauthorized.withHeaders("WWW-Authenticate" -> """Basic realm="Secured"""") - - // derived from https://gist.github.com/guillaumebort/2328236 - def credentialsFromAuth(authorization: String): Option[Credentials] = { - authorization.split(" ").drop(1).headOption.flatMap { encoded => - new String(decodeBase64(encoded.getBytes)).split(":").toList match { - case u :: p :: Nil => Some(Credentials(u, p)) - case _ => None - } - } - } - - def validUserFor[U](req: RequestHeader, userForCredentials: Credentials => Option[U]): Option[U] = for { - authorizationHeader <- req.headers.get("Authorization") - credentials <- credentialsFromAuth(authorizationHeader) - validUser <- userForCredentials(credentials) - } yield validUser - - def basicAuth[U](userForCredentials: Credentials => Option[U]) = - new AuthenticatedBuilder(req => validUserFor(req, userForCredentials), _ => requestBasicAuth) - } \ No newline at end of file diff --git a/app/lib/actions/Parsers.scala b/app/lib/actions/Parsers.scala index 72b27a6..63a2d52 100644 --- a/app/lib/actions/Parsers.scala +++ b/app/lib/actions/Parsers.scala @@ -1,41 +1,11 @@ package lib.actions -import javax.crypto.Mac -import javax.crypto.spec.SecretKeySpec - import com.madgag.scalagithub.model.RepoId -import play.api.Logger -import play.api.libs.iteratee.{Iteratee, Traversable} -import play.api.libs.json.{JsValue, Json} -import play.api.libs.{Codecs, Crypto} -import play.api.mvc.{BodyParser, RequestHeader, Result, Results} - -import scala.concurrent.Future -import scala.util.control.NonFatal - -object RepoSecretKey { - def sharedSecretForRepo(repo: RepoId) = { - val signature = Crypto.sign("GitHub-Repo:"+repo.fullName) - Logger.debug(s"Repo $repo signature $signature") - signature - } -} +import play.api.libs.json.JsValue object Parsers { - def assertSecureEquals(s1: String, s2: String) = { - assert(Crypto.constantTimeEquals(s1, s2), "HMAC signatures did not match") - Logger.debug("HMAC Signatures matched!") - } - def parseGitHubHookJson(jsValue: JsValue): RepoId = (jsValue \ "repository" \ "full_name").validate[String].map(RepoId.from).get - type FullBodyParser[+A] = (RequestHeader, Array[Byte]) => Either[Result, A] - - def sign(message: Array[Byte], key: Array[Byte]): String = { - val mac = Mac.getInstance("HmacSHA1") - mac.init(new SecretKeySpec(key, "HmacSHA1")) - Codecs.toHexString(mac.doFinal(message)) - } } diff --git a/app/lib/gitgithub/IssueUpdater.scala b/app/lib/gitgithub/IssueUpdater.scala deleted file mode 100644 index fd99b1f..0000000 --- a/app/lib/gitgithub/IssueUpdater.scala +++ /dev/null @@ -1,67 +0,0 @@ -package lib.gitgithub - -import com.madgag.scalagithub.GitHub._ -import com.madgag.scalagithub.model.{PullRequest, Repo} -import lib.{Bot, Delayer, LabelledState, RepoSnapshot} -import play.api.Logger - -import scala.concurrent.ExecutionContext.Implicits.global -import scala.concurrent.Future - -object IssueUpdater { - val logger = Logger(getClass) -} - -/** - * - * @tparam IssueType Pull Request or Issue - * @tparam PersistableState State that can be converted to and from GitHub issue labels, ie a set of Strings - * @tparam Snapshot A present-state snapshot that can yield a PersistableState - */ -trait IssueUpdater[IssueType <: PullRequest, PersistableState, Snapshot <: StateSnapshot[PersistableState]] { - - implicit val github = Bot.github - - val repo: Repo - - val repoSnapshot: RepoSnapshot - - val labelToStateMapping:LabelMapping[PersistableState] - - def ignoreItemsWithExistingState(existingState: PersistableState): Boolean - - def snapshot(oldState: PersistableState, issue: IssueType): Future[Snapshot] - - def actionTaker(snapshot: Snapshot) - - def process(issueLike: IssueType): Future[Option[Snapshot]] = { - logger.trace(s"handling ${issueLike.prId.slug}") - for { - oldLabels <- new LabelledState(issueLike, repoSnapshot.allPossibleCheckpointPRLabels).currentLabelsF - snapshot <- takeSnapshotOf(issueLike, oldLabels) - } yield snapshot - } - - def takeSnapshotOf(issueLike: IssueType, oldLabels: Set[String]): Future[Option[Snapshot]] = { - val existingPersistedState: PersistableState = labelToStateMapping.stateFrom(oldLabels) - if (!ignoreItemsWithExistingState(existingPersistedState)) { - for (currentSnapshot <- snapshot(existingPersistedState, issueLike)) yield { - val newPersistableState = currentSnapshot.newPersistableState - val stateChanged = newPersistableState != existingPersistedState - - logger.debug(s"handling ${issueLike.prId.slug} : state: existing=$existingPersistedState new=$newPersistableState stateChanged=$stateChanged") - - if (stateChanged) { - logger.info(s"#${issueLike.prId.slug} state-change: $existingPersistedState -> $newPersistableState") - val newLabels: Set[String] = labelToStateMapping.labelsFor(newPersistableState) - assert(oldLabels != newLabels, s"Labels should differ for differing states. labels=$oldLabels oldState=$existingPersistedState newState=$newPersistableState") - issueLike.labels.replace(newLabels.toSeq) - Delayer.doAfterSmallDelay { - actionTaker(currentSnapshot) - } - } - Some(currentSnapshot) - } - } else Future.successful(None) - } -} diff --git a/app/lib/gitgithub/package.scala b/app/lib/gitgithub/package.scala new file mode 100644 index 0000000..57da258 --- /dev/null +++ b/app/lib/gitgithub/package.scala @@ -0,0 +1,13 @@ +package lib + +import akka.NotUsed +import akka.stream.Materializer +import akka.stream.scaladsl.{Keep, Sink} + +import scala.concurrent.Future + +package object gitgithub { + implicit class RichSource[T](s: akka.stream.scaladsl.Source[Seq[T], NotUsed]) { + def all()(implicit mat: Materializer): Future[Seq[T]] = s.toMat(Sink.reduce[Seq[T]](_ ++ _))(Keep.right).run() + } +} diff --git a/app/lib/librato/LibratoApiClient.scala b/app/lib/librato/LibratoApiClient.scala index f8ada4c..a76a8c1 100644 --- a/app/lib/librato/LibratoApiClient.scala +++ b/app/lib/librato/LibratoApiClient.scala @@ -1,11 +1,12 @@ package lib.librato import com.madgag.okhttpscala._ -import com.netaporter.uri.Uri +import io.lemonlabs.uri.Uri import com.typesafe.scalalogging.LazyLogging import lib.librato.model.Annotation import okhttp3.Request.Builder import okhttp3._ +import play.api.Configuration import play.api.libs.json.Json.{stringify, toJson} import scala.concurrent.ExecutionContext.Implicits.global @@ -38,12 +39,9 @@ class LibratoApiClient(username: String, token: String) extends LazyLogging { object LibratoApiClient { - import play.api.Play.current - val config = play.api.Play.configuration - - val instanceOpt = for { - userId <- config.getString("librato.userId") - token <- config.getString("librato.token") + def instanceOptFrom(config: Configuration): Option[LibratoApiClient] = for { + userId <- config.getOptional[String]("librato.userId") + token <- config.getOptional[String]("librato.token") } yield new LibratoApiClient(userId,token) } diff --git a/app/lib/librato/LibratoDeployReporter.scala b/app/lib/librato/LibratoDeployReporter.scala new file mode 100644 index 0000000..298daf0 --- /dev/null +++ b/app/lib/librato/LibratoDeployReporter.scala @@ -0,0 +1,41 @@ +package lib.librato + +import com.madgag.scalagithub.model.PullRequest +import io.lemonlabs.uri.Uri +import lib.Responsibility.createdByAndMergedByFor +import lib.labels.Seen +import lib.librato.model.{Annotation, Link} +import lib.{EverythingYouWantToKnowAboutACheckpoint, PullRequestCheckpointsStateChangeSummary, RepoLevelDetails, RepoSnapshot, UpdateReporter} + +import scala.concurrent.{ExecutionContext, Future} + +class LibratoDeployReporter( + librato: LibratoApiClient +)(implicit + ec: ExecutionContext +) extends UpdateReporter { + + override def report( + repoSnapshot: RepoSnapshot, + pr: PullRequest, + checkpointsChangeSummary: PullRequestCheckpointsStateChangeSummary + ): Unit = Future.traverse(checkpointsChangeSummary.changedByState(Seen)) { checkpoint => + report(pr, checkpoint) + } + + private def report(pr: PullRequest, checkpoint: EverythingYouWantToKnowAboutACheckpoint): Future[_] = { + librato.createAnnotation(s"${pr.baseRepo.name}.prout", Annotation( + title = s"PR #${pr.number} : '${pr.title}' deployed", + description = Some(createdByAndMergedByFor(pr).capitalize), + start_time = pr.merged_at.map(_.toInstant), + end_time = Some(checkpoint.snapshot.time), + source = Some(checkpoint.snapshot.checkpoint.name), + links = Seq(Link( + rel = "github", + label = Some(s"PR #${pr.number}"), + href = Uri.parse(pr.html_url) + )) + )) + } + +} diff --git a/app/lib/librato/model/Librato.scala b/app/lib/librato/model/Librato.scala index 8bd54c2..a94d728 100644 --- a/app/lib/librato/model/Librato.scala +++ b/app/lib/librato/model/Librato.scala @@ -2,7 +2,7 @@ package lib.librato.model import java.time.Instant -import com.netaporter.uri.Uri +import io.lemonlabs.uri.Uri import play.api.libs.json.{JsNumber, JsString, Json, Writes} case class Link( diff --git a/app/lib/package.scala b/app/lib/package.scala deleted file mode 100644 index 2abe52a..0000000 --- a/app/lib/package.scala +++ /dev/null @@ -1,10 +0,0 @@ -import lib.Config.Checkpoint -import org.eclipse.jgit.lib.AbbreviatedObjectId - -import scala.concurrent.Future - -package object lib { - - type CheckpointSnapshoter = Checkpoint => Future[Iterator[AbbreviatedObjectId]] - -} diff --git a/app/lib/sentry/SentryApiClient.scala b/app/lib/sentry/SentryApiClient.scala index ae94e79..42c0d1f 100644 --- a/app/lib/sentry/SentryApiClient.scala +++ b/app/lib/sentry/SentryApiClient.scala @@ -1,11 +1,13 @@ package lib.sentry import com.madgag.okhttpscala._ -import com.netaporter.uri.Uri +import io.lemonlabs.uri.Uri import com.typesafe.scalalogging.LazyLogging +import lib.librato.LibratoApiClient import lib.sentry.model.CreateRelease import okhttp3.Request.Builder import okhttp3._ +import play.api.Configuration import play.api.libs.json.Json.{stringify, toJson} import scala.concurrent.ExecutionContext.Implicits.global @@ -35,15 +37,9 @@ class SentryApiClient(token: String , val org: String) extends LazyLogging { } - object SentryApiClient { - - import play.api.Play.current - val config = play.api.Play.configuration - - lazy val instanceOpt = for { - org <- config.getString("sentry.org") - token <- config.getString("sentry.token") + def instanceOptFrom(config: Configuration): Option[SentryApiClient] = for { + org <- config.getOptional[String]("sentry.org") + token <- config.getOptional[String]("sentry.token") } yield new SentryApiClient(token,org) - } diff --git a/app/lib/sentry/SentryReporter.scala b/app/lib/sentry/SentryReporter.scala new file mode 100644 index 0000000..9c2e4cf --- /dev/null +++ b/app/lib/sentry/SentryReporter.scala @@ -0,0 +1,55 @@ +package lib.sentry + +import com.madgag.git._ +import com.madgag.scalagithub.model.PullRequest +import io.lemonlabs.uri.Uri +import lib.sentry.model.CreateRelease +import lib.{PullRequestCheckpointsStateChangeSummary, RepoLevelDetails, RepoSnapshot, UpdateReporter} +import org.eclipse.jgit.revwalk.RevWalk +import play.api.Logging + +import scala.concurrent.Future + +class SentryReporter( + sentry: SentryApiClient +) extends UpdateReporter with Logging { + override def report(repoSnapshot: RepoSnapshot, pr: PullRequest, checkpointsChangeSummary: PullRequestCheckpointsStateChangeSummary): Unit = { + if (checkpointsChangeSummary.newlyMerged) { + logger.info(s"action taking: ${pr.prId} is newly merged") + val repoLevelDetails = repoSnapshot.repoLevelDetails + + for { + sentryRelease <- sentryReleaseOption(repoSnapshot, pr) + } { + val ref = lib.sentry.model.Ref( + repoLevelDetails.repo.repoId, + sentryRelease.mergeCommit, + sentryRelease.mergeCommit.asRevCommit(new RevWalk(repoLevelDetails.gitRepo.getObjectDatabase.threadLocalResources.reader())).getParents.headOption) + + logger.info(s"${pr.prId.slug} : ref=$ref") + + sentry.createRelease(CreateRelease( + sentryRelease.version, + Some(sentryRelease.version), + Some(Uri.parse(pr.html_url)), + sentryRelease.projects, + refs=Seq(ref) + )) + } + } + } + + private def sentryReleaseOption(repoSnapshot: RepoSnapshot, pr: PullRequest): Option[PRSentryRelease] = { + val sentryProjects = for { + configs <- repoSnapshot.activeConfByPullRequest.get(pr).toSeq + config <- configs + sentryConf <- config.sentry.toSeq + sentryProject <- sentryConf.projects + } yield sentryProject + + val sentryReleaseOpt = for { + mergeCommit <- pr.merge_commit_sha if sentryProjects.nonEmpty + } yield PRSentryRelease(mergeCommit, sentryProjects) + sentryReleaseOpt + } +} diff --git a/app/lib/sentry/model/Sentry.scala b/app/lib/sentry/model/Sentry.scala index cc0a308..490812a 100644 --- a/app/lib/sentry/model/Sentry.scala +++ b/app/lib/sentry/model/Sentry.scala @@ -2,7 +2,7 @@ package lib.sentry.model import java.time.Instant -import com.netaporter.uri.Uri +import io.lemonlabs.uri.Uri import org.eclipse.jgit.lib.ObjectId import play.api.libs.json.{JsString, Json, Writes} import Sentry._ diff --git a/app/lib/slack/DeployReporter.scala b/app/lib/slack/DeployReporter.scala index 2424947..e936f14 100644 --- a/app/lib/slack/DeployReporter.scala +++ b/app/lib/slack/DeployReporter.scala @@ -1,47 +1,62 @@ package lib.slack -import com.netaporter.uri.Uri -import com.netaporter.uri.dsl._ -import lib.PullRequestCheckpointsStateChangeSummary +import cats.data.NonEmptySeq +import com.madgag.scalagithub.model.{PullRequest, User} +import io.lemonlabs.uri.Url import lib.labels.Seen -import play.api.Logger -import play.api.Play.current -import play.api.libs.concurrent.Execution.Implicits.defaultContext -import play.api.libs.json.Json -import play.api.libs.ws.WS - -object DeployReporter { - - def report(snapshot: PullRequestCheckpointsStateChangeSummary, hooks: Seq[Uri]) { - val slackHooks = hooks.filter(_.host.contains("hooks.slack.com")) - if (slackHooks.nonEmpty) { - for (changedSnapshots <- snapshot.changedByState.get(Seen)) { - val pr = snapshot.prCheckpointDetails.pr - val mergedBy = pr.merged_by.get - val checkpoints = changedSnapshots.map(_.snapshot.checkpoint) - val attachments = Seq(Attachment(s"PR #${pr.number} deployed to ${checkpoints.map(_.name).mkString(", ")}", - Seq( - Attachment.Field("PR", s"<${pr.html_url}|#${pr.number}>", short = true), - Attachment.Field("Merged by", s"<${mergedBy.html_url}|${mergedBy.atLogin}>", short = true) - ) - )) - - val checkpointsAsSlack = checkpoints.map(c => s"<${c.details.url}|${c.name}>").mkString(", ") - val json = Json.toJson( - Message( - s"*Deployed to $checkpointsAsSlack: ${pr.title}*", - Some(lib.Bot.user.login), - Some(mergedBy.avatar_url), - attachments - ) - ) - - for (hook <- slackHooks) { - WS.url(hook).post(json).onComplete { - r => Logger.debug(s"Response from Slack: ${r.map(_.body)}") - } - } - } +import lib._ +import play.api.Logging +import play.api.libs.json.{JsValue, Json} +import play.api.libs.ws.WSClient + +import scala.concurrent.{ExecutionContext, Future} + +class DeployReporter( + ws: WSClient, + bot: Bot +)(implicit + ec: ExecutionContext +) extends Logging with UpdateReporter { + + override def report( + repoSnapshot: RepoSnapshot, + pr: PullRequest, + checkpointsChangeSummary: PullRequestCheckpointsStateChangeSummary + ): Unit = slackHooksFrom(repoSnapshot.repoLevelDetails).flatMap { slackHooks => + report(checkpointsChangeSummary, slackHooks) + }.getOrElse(Future.successful(())) + + private def slackHooksFrom(repoLevelDetails: RepoLevelDetails): Option[NonEmptySeq[Url]] = + NonEmptySeq.fromSeq(repoLevelDetails.hooks.filter(_.hostOption.exists(_.value == "hooks.slack.com"))) + + private def report(checkpointChangeSummary: PullRequestCheckpointsStateChangeSummary, slackHooks: NonEmptySeq[Url]): Option[Future[Unit]] = { + val pr = checkpointChangeSummary.prCheckpointDetails.pr + for (changedSnapshots <- checkpointChangeSummary.changedByState.get(Seen)) yield { + val json = messageJsonFor(pr, changedSnapshots) + + Future.traverse(slackHooks.toSeq) { hook => + val responseF = ws.url(hook.toString).post(json) + responseF.onComplete { r => logger.debug(s"Response from Slack: ${r.map(_.body)}") } + responseF + }.map(_ => ()) } } + + private def messageJsonFor(pr: PullRequest, changedSnapshots: Set[EverythingYouWantToKnowAboutACheckpoint]): JsValue = { + val checkpointsWherePRIsNewlySeen = changedSnapshots.map(_.snapshot.checkpoint) + + Json.toJson( + Message( + s"*Deployed to ${checkpointsWherePRIsNewlySeen.map(slackLinkFor).mkString(", ")}: ${pr.title}*", + Some(bot.user.login), + pr.merged_by.map(_.avatar_url), + attachments = Seq(Attachment(s"PR #${pr.number} deployed to ${checkpointsWherePRIsNewlySeen.map(_.name).mkString(", ")}", Seq( + Attachment.Field("PR", s"<${pr.html_url}|#${pr.number}>", short = true), + ) ++ pr.merged_by.map(mergedBy => Attachment.Field("Merged by", slackLinkFor(mergedBy), short = true)))) + ) + ) + } + + private def slackLinkFor(c: Config.Checkpoint) = s"<${c.details.url}|${c.name}>" + private def slackLinkFor(user: User) = s"<${user.html_url}|${user.atLogin}>" } diff --git a/app/monitoring/SentryLogging.scala b/app/monitoring/SentryLogging.scala index 80b0ed9..6a75f3c 100644 --- a/app/monitoring/SentryLogging.scala +++ b/app/monitoring/SentryLogging.scala @@ -7,33 +7,31 @@ import com.getsentry.raven.dsn.Dsn import com.getsentry.raven.logback.SentryAppender import org.slf4j.Logger.ROOT_LOGGER_NAME import org.slf4j.LoggerFactory -import play.api -import play.api.Play.configuration +import play.api.Logging -object SentryLogging { - import play.api.Play.current +class SentryLogging( + config: play.api.Configuration +) extends Logging { - val dsnOpt = configuration.getString("sentry.dsn").map(new Dsn(_)) + val dsnOpt: Option[Dsn] = config.getOptional[String]("sentry.dsn").map(new Dsn(_)) - def init() { - dsnOpt match { - case None => - api.Logger.warn("No Sentry logging configured (OK for dev)") - case Some(dsn) => - api.Logger.info(s"Initialising Sentry logging for ${dsn.getHost}") - val tags = Map("gitCommitId" -> app.BuildInfo.gitCommitId) - val tagsString = tags.map { case (key, value) => s"$key:$value" }.mkString(",") + def init(): Unit = dsnOpt match { + case None => + logger.warn("No Sentry logging configured (OK for dev)") + case Some(dsn) => + logger.info(s"Initialising Sentry logging for ${dsn.getHost}") + val tags = Map("gitCommitId" -> app.BuildInfo.gitCommitId) + val tagsString = tags.map { case (key, value) => s"$key:$value" }.mkString(",") - val filter = new ThresholdFilter { setLevel("ERROR") } - filter.start() // OMG WHY IS THIS NECESSARY LOGBACK? + val filter = new ThresholdFilter { setLevel("ERROR") } + filter.start() // OMG WHY IS THIS NECESSARY LOGBACK? - val sentryAppender = new SentryAppender(ravenInstance(dsn)) { - addFilter(filter) - setTags(tagsString) - setContext(LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext]) - } - sentryAppender.start() - LoggerFactory.getLogger(ROOT_LOGGER_NAME).asInstanceOf[Logger].addAppender(sentryAppender) - } + val sentryAppender = new SentryAppender(ravenInstance(dsn)) { + addFilter(filter) + setTags(tagsString) + setContext(LoggerFactory.getILoggerFactory.asInstanceOf[LoggerContext]) + } + sentryAppender.start() + LoggerFactory.getLogger(ROOT_LOGGER_NAME).asInstanceOf[Logger].addAppender(sentryAppender) } } diff --git a/app/views/userPages/index.scala.html b/app/views/userPages/index.scala.html index 04a295f..c91b41e 100644 --- a/app/views/userPages/index.scala.html +++ b/app/views/userPages/index.scala.html @@ -1,4 +1,4 @@ -@()(implicit req: Request[AnyContent]) +@()(implicit req: Request[AnyContent], bot: lib.Bot) @main {
@@ -11,7 +11,7 @@

What is prout?

Setup

Follow the 4-step program:

    -
  1. Give @lib.Bot.user.atLogin write-access to your repo (so it can set labels on your pull request)
    2. +
  2. Give @bot.user.atLogin write-access to your repo (so it can set labels on your pull request)
  3. Add one or more .prout.json config files to your project
  4. Add callbacks to prout - ie a GitHub webhook and ideally also a post-deploy hook
  5. Expose the commit id of your build on your deployed site
    6. diff --git a/app/views/userPages/repo.scala.html b/app/views/userPages/repo.scala.html index bac5a8b..5f753f1 100644 --- a/app/views/userPages/repo.scala.html +++ b/app/views/userPages/repo.scala.html @@ -2,9 +2,15 @@ @import org.eclipse.jgit.lib.ObjectId @import scala.util.{Failure, Success} -@(proutPresenceQuickCheck: Boolean, repoSnapshot: lib.RepoSnapshot, diagnostic: lib.Diagnostic)(implicit req: Request[AnyContent]) +@import lib.sentry.SentryApiClient +@( + proutPresenceQuickCheck: Boolean, + repoSnapshot: lib.RepoSnapshot, + diagnostic: lib.Diagnostic, + sentryApiClientOpt: Option[SentryApiClient] +)(implicit req: Request[AnyContent], bot: lib.Bot) @showCommit(commitId: ObjectId) = {@commitId.name.take(7)} - @showCommits(commitIds: Traversable[ObjectId]) = {@for(commitId<-commitIds) {@showCommit(commitId) }} + @showCommits(commitIds: Iterable[ObjectId]) = {@for(commitId<-commitIds) {@showCommit(commitId) }} @configLink(folder: String) = {@folder@ProutConfigFileName} @main { @@ -13,7 +19,7 @@
    @for(permissions <- repoSnapshot.repo.permissions) { -

    Permissions for @lib.Bot.user.atLogin

    +

    Permissions for @bot.user.atLogin

    • push @if(permissions.push) { @@ -88,7 +94,7 @@
      Sentry
    • @sentryProject
      • }
    - @if(lib.sentry.SentryApiClient.instanceOpt.isEmpty) { + @if(sentryApiClientOpt.isEmpty) { ...but no Sentry credentials are available! } } diff --git a/build.sbt b/build.sbt index b09a6fb..fdb633a 100644 --- a/build.sbt +++ b/build.sbt @@ -2,17 +2,15 @@ name := "prout" version := "1.0-SNAPSHOT" -scalaVersion := "2.11.12" +scalaVersion := "2.13.10" updateOptions := updateOptions.value.withCachedResolution(true) -resolvers += Resolver.sonatypeRepo("releases") +resolvers ++= Resolver.sonatypeOssRepos("releases") buildInfoKeys := Seq[BuildInfoKey]( name, - BuildInfoKey.constant("gitCommitId", Option(System.getenv("SOURCE_VERSION")) getOrElse(try { - "git rev-parse HEAD".!!.trim - } catch { case e: Exception => "unknown" })) + "gitCommitId" -> Option(System.getenv("SOURCE_VERSION")).getOrElse("unknown") ) buildInfoPackage := "app" @@ -20,25 +18,24 @@ buildInfoPackage := "app" lazy val root = (project in file(".")).enablePlugins(PlayScala, BuildInfoPlugin) libraryDependencies ++= Seq( - cache, filters, ws, - "com.typesafe.akka" %% "akka-agent" % "2.3.2", + "com.softwaremill.macwire" %% "macros" % "2.5.7" % Provided, // slight finesse: 'provided' as only used for compile + "com.madgag" %% "scala-collection-plus" % "0.11", + "org.typelevel" %% "cats-core" % "2.7.0", + "com.github.blemale" %% "scaffeine" % "5.2.0", "org.webjars" % "bootstrap" % "3.3.2-1", - "com.getsentry.raven" % "raven-logback" % "8.0.2", - "com.github.nscala-time" %% "nscala-time" % "2.16.0", - "com.netaporter" %% "scala-uri" % "0.4.16", - "com.typesafe.scala-logging" %% "scala-logging" % "3.5.0", - "com.github.scala-incubator.io" %% "scala-io-file" % "0.4.3-1", + "com.getsentry.raven" % "raven-logback" % "8.0.3", + "com.github.nscala-time" %% "nscala-time" % "2.30.0", + "io.lemonlabs" %% "scala-uri" % "4.0.2", + "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5", "org.webjars.bower" % "octicons" % "3.1.0", - "com.madgag" %% "play-git-hub" % "4.5", - "com.madgag.scala-git" %% "scala-git-test" % "3.0" % "test", - "org.scalatestplus" %% "play" % "1.4.0" % "test" + "com.madgag" %% "play-git-hub" % "5.0", + "com.madgag.scala-git" %% "scala-git-test" % "4.3" % Test, + "org.scalatestplus.play" %% "scalatestplus-play" % "5.1.0" % Test ) routesImport ++= Seq("com.madgag.scalagithub.model._","com.madgag.playgithub.Binders._") -sources in (Compile,doc) := Seq.empty - -publishArtifact in (Compile, packageDoc) := false - +Compile/doc/sources := Seq.empty +Compile/packageDoc/publishArtifact := false \ No newline at end of file diff --git a/conf/application.conf b/conf/application.conf index 7ae8b4e..f48bd79 100644 --- a/conf/application.conf +++ b/conf/application.conf @@ -1,8 +1,13 @@ # This is the main configuration file for the application. # ~~~~~ -# https://www.playframework.com/documentation/2.4.x/ApplicationSecret -play.crypto.secret=${?APPLICATION_SECRET} +play.application.loader=configuration.AppLoader + +# Secret key +# ~~~~~ +# The secret key is used to for signing session cookies and CSRF tokens. +# If you deploy your application to several instances be sure to use the same key! +play.http.secret.key=${?APPLICATION_SECRET} sentry.dsn=${?SENTRY_DSN} @@ -11,7 +16,7 @@ sentry.dsn=${?SENTRY_DSN} application.langs="en" github { - access.token=${PROUT_GITHUB_ACCESS_TOKEN} + botAccessToken=${PROUT_GITHUB_ACCESS_TOKEN} clientId=${?GITHUB_APP_CLIENT_ID} clientSecret=${?GITHUB_APP_CLIENT_SECRET} diff --git a/conf/logback.xml b/conf/logback.xml index 0374c6c..3a47bd7 100644 --- a/conf/logback.xml +++ b/conf/logback.xml @@ -1,6 +1,6 @@ - + logs/application.log diff --git a/conf/routes b/conf/routes index f79ba47..ba86a45 100644 --- a/conf/routes +++ b/conf/routes @@ -1,18 +1,13 @@ ### Callback endpoints - POST /api/update/$repo<[^/]+/[^/]+> controllers.Api.updateRepo(repo: RepoId) # GET because that's what RiffRaff supports... GET /api/update/$repo<[^/]+/[^/]+> controllers.Api.updateRepo(repo: RepoId) POST /api/hooks/github controllers.Api.githubHook() - - - - ### Non-API endpoints - resources for humans to view GET / controllers.Application.index() -GET /view/$repo<[^/]+/[^/]+> controllers.Application.zoomba(repo: RepoId) +GET /view/$repo<[^/]+/[^/]+> controllers.Application.configDiagnostic(repo: RepoId) GET /assets/*file controllers.Assets.at(path="/public", file) GET /oauth/callback controllers.Auth.oauthCallback(code: String) diff --git a/project/build.properties b/project/build.properties index 8e682c5..8b9a0b0 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=0.13.18 +sbt.version=1.8.0 diff --git a/project/plugins.sbt b/project/plugins.sbt index 3f7aa13..274467c 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,6 +1,13 @@ -// Use the Play sbt plugin for Play projects -addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.4.11") +addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.8.18") -addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.7.0") +addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.11.0") -addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.8.2") +/* + scala-xml has been updated to 2.x in sbt, but not in other sbt plugins like sbt-native-packager + See: https://github.com/scala/bug/issues/12632 + This is effectively overrides the safeguards (early-semver) put in place by the library authors ensuring binary compatibility. + We consider this a safe operation because when set under `projects/` (ie *not* in `build.sbt` itself) it only affects the + compilation of build.sbt, not of the application build itself. + Once the build has succeeded, there is no further risk (ie of a runtime exception due to clashing versions of `scala-xml`). + */ +libraryDependencySchemes += "org.scala-lang.modules" %% "scala-xml" % VersionScheme.Always \ No newline at end of file diff --git a/system.properties b/system.properties index 965f453..5e87ec7 100644 --- a/system.properties +++ b/system.properties @@ -1,2 +1,2 @@ -java.runtime.version=1.8 +java.runtime.version=11 diff --git a/test/FunctionalSpec.scala b/test/FunctionalSpec.scala index 93be9f0..a8464a5 100644 --- a/test/FunctionalSpec.scala +++ b/test/FunctionalSpec.scala @@ -1,18 +1,11 @@ -import java.util.Base64 - -import com.madgag.scalagithub.GitHub._ -import com.madgag.scalagithub.commands.{CreateFile, CreateRef} -import com.madgag.scalagithub.model.{Repo, RepoId} import lib.RepoSnapshot.ClosedPRsMostlyRecentlyUpdated import lib._ +import lib.gitgithub.RichSource import org.eclipse.jgit.lib.ObjectId.zeroId import org.scalatest.Inside -import org.scalatest.concurrent.PatienceConfiguration.Timeout -import play.api.libs.json.{JsDefined, JsString, JsSuccess} import scala.concurrent.ExecutionContext.Implicits.global -import scala.concurrent.duration._ -import scala.util.Random +import scala.language.postfixOps class FunctionalSpec extends Helpers with TestRepoCreation with Inside { diff --git a/test/lib/CheckpointSpec.scala b/test/lib/CheckpointSpec.scala index 15559fa..8cbf7af 100644 --- a/test/lib/CheckpointSpec.scala +++ b/test/lib/CheckpointSpec.scala @@ -1,6 +1,6 @@ package lib -import com.netaporter.uri.dsl._ +import io.lemonlabs.uri.typesafe.dsl._ import lib.Config.{Checkpoint, CheckpointDetails} import org.joda.time.Period import org.scalatest.concurrent.{Eventually, IntegrationPatience, ScalaFutures} @@ -11,10 +11,9 @@ class CheckpointSpec extends PlaySpec with ScalaFutures with Eventually with Int "Checkpoint snapshots" must { "be able to hit Ophan" in { - val checkpoint = Checkpoint("PROD", CheckpointDetails("https://dashboard.ophan.co.uk/login", Period.parse("PT1H"))) - whenReady(CheckpointSnapshot(checkpoint)) { s => - s must not be 'empty - } + val checkpoint = + Checkpoint("PROD", CheckpointDetails("https://dashboard.ophan.co.uk/login", Period.parse("PT1H"))) + whenReady(CheckpointSnapshoter.snapshot(checkpoint)) { _ must not be empty } } } } \ No newline at end of file diff --git a/test/lib/ConfigSpec.scala b/test/lib/ConfigSpec.scala index c427e05..540d6cd 100644 --- a/test/lib/ConfigSpec.scala +++ b/test/lib/ConfigSpec.scala @@ -1,6 +1,6 @@ package lib -import com.netaporter.uri.Uri +import io.lemonlabs.uri.Uri import lib.Config.{CheckpointDetails, CheckpointMessages} import lib.labels.{Overdue, Seen} import org.joda.time.Period.minutes @@ -8,8 +8,6 @@ import org.scalatest.{Inside, OptionValues} import org.scalatestplus.play._ import play.api.libs.json._ -import scalax.io.JavaConverters._ - class ConfigSpec extends PlaySpec with OptionValues with Inside { "Config json parsing" must { @@ -50,6 +48,6 @@ class ConfigSpec extends PlaySpec with OptionValues with Inside { } def checkpointDetailsFrom(resourcePath: String): JsResult[CheckpointDetails] = { - Json.parse(getClass.getResource(resourcePath).asInput.byteArray).validate[CheckpointDetails] + Json.parse(getClass.getResourceAsStream(resourcePath)).validate[CheckpointDetails] } } \ No newline at end of file diff --git a/test/lib/Helpers.scala b/test/lib/Helpers.scala index dddd6c6..fb477c7 100644 --- a/test/lib/Helpers.scala +++ b/test/lib/Helpers.scala @@ -1,38 +1,46 @@ package lib -import java.net.URL -import com.madgag.scalagithub.GitHub._ import com.madgag.scalagithub.commands.{CreatePullRequest, MergePullRequest} import com.madgag.scalagithub.model._ import com.madgag.scalagithub.{GitHub, GitHubCredentials} +import lib.gitgithub.RichSource +import lib.sentry.SentryApiClient import org.eclipse.jgit.lib.{AbbreviatedObjectId, ObjectId} -import org.scalatest.{Inside, Inspectors} import org.scalatest.concurrent.{Eventually, ScalaFutures} import org.scalatest.time.{Millis, Seconds, Span} +import org.scalatest.{Inside, Inspectors} import org.scalatestplus.play._ -import play.api.Logger +import org.scalatestplus.play.components.OneAppPerSuiteWithComponents +import play.api.routing.Router +import play.api.{BuiltInComponents, BuiltInComponentsFromContext, Logger, NoHttpFiltersComponents} +import java.net.URL +import java.nio.file.Files import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future -import scalax.file.ImplicitConversions._ -import scalax.file.Path case class PRText(title: String, desc: String) -trait Helpers extends PlaySpec with OneAppPerSuite with Inspectors with ScalaFutures with Eventually with Inside { +trait Helpers extends PlaySpec with OneAppPerSuiteWithComponents with Inspectors with ScalaFutures with Eventually with Inside { val logger = Logger(getClass) + override def components: BuiltInComponents = new BuiltInComponentsFromContext(context) with NoHttpFiltersComponents { + + override lazy val router: Router = Router.empty + } implicit override val patienceConfig = PatienceConfig(timeout = scaled(Span(12, Seconds)), interval = scaled(Span(850, Millis))) val githubToken = sys.env("PROUT_GITHUB_ACCESS_TOKEN") - val githubCredentials = GitHubCredentials.forAccessKey(githubToken, Path.createTempDirectory().toPath).get + val githubCredentials = + GitHubCredentials.forAccessKey(githubToken, Files.createTempDirectory("tmpDirPrefix")).get val slackWebhookUrlOpt = sys.env.get("PROUT_TEST_SLACK_WEBHOOK").map(new URL(_)) implicit lazy val github = new GitHub(githubCredentials) + implicit lazy val mat = app.materializer def labelsOn(pr: PullRequest): Set[String] = pr.labels.list().all().futureValue.map(_.name).toSet @@ -56,15 +64,15 @@ trait Helpers extends PlaySpec with OneAppPerSuite with Inspectors with ScalaFut var checkpointCommitFuture: Future[Iterator[AbbreviatedObjectId]] = Future.successful(Iterator.empty) - def setCheckpointTo(commitId: AbbreviatedObjectId) { + def setCheckpointTo(commitId: AbbreviatedObjectId): Unit = { checkpointCommitFuture = Future.successful(Iterator(commitId)) } - def setCheckpointTo(objectId: ObjectId) { + def setCheckpointTo(objectId: ObjectId): Unit = { setCheckpointTo(AbbreviatedObjectId.fromObjectId(objectId)) } - def setCheckpointTo(branchName: String) { + def setCheckpointTo(branchName: String): Unit = { val objectId = githubRepo.refs.get(s"heads/$branchName").futureValue.objectId setCheckpointTo(objectId) logger.info(s"Set checkpoint to '$branchName' (${objectId.name.take(8)})") @@ -72,18 +80,31 @@ trait Helpers extends PlaySpec with OneAppPerSuite with Inspectors with ScalaFut def setCheckpointToMatchDefaultBranch = setCheckpointTo(githubRepo.default_branch) - def setCheckpointFailureTo(exception: Exception) { + def setCheckpointFailureTo(exception: Exception): Unit = { checkpointCommitFuture = Future.failed(exception) } - val checkpointSnapshoter: CheckpointSnapshoter = _ => checkpointCommitFuture + implicit val checkpointSnapshoter: CheckpointSnapshoter = _ => checkpointCommitFuture + implicit val sentryApiClient: Option[SentryApiClient] = None + + val delayer = new Delayer(app.actorSystem) + + val bot: Bot = Bot.forAccessToken(githubToken) + + val repoSnapshotFactory: RepoSnapshot.Factory = new RepoSnapshot.Factory(bot) - val scheduler = new ScanScheduler(githubRepo.repoId, checkpointSnapshoter, github) + val droid: Droid = new Droid( + repoSnapshotFactory, + new RepoUpdater(), + new PRUpdater(delayer) + ) + + val scheduler = new ScanScheduler(githubRepo.repoId, droid, actorSystem = app.actorSystem, delayer) override val toString: String = pr.html_url } - def scan[T](shouldAddComment: Boolean)(issueFun: PullRequest => T)(implicit repoPR: RepoPR) { + def scan[T](shouldAddComment: Boolean)(issueFun: PullRequest => T)(implicit repoPR: RepoPR): Unit = { val commentsBeforeScan = repoPR.listComments() whenReady(repoPR.scheduler.scan()) { s => eventually { @@ -96,18 +117,7 @@ trait Helpers extends PlaySpec with OneAppPerSuite with Inspectors with ScalaFut } } - def scanUntil[T](shouldAddComment: Boolean)(issueFun: PullRequest => T)(implicit repoPR: RepoPR) { - val commentsBeforeScan = repoPR.listComments() - eventually { - whenReady(repoPR.scheduler.scan()) { s => - val commentsAfterScan = repoPR.listComments() - commentsAfterScan must have size (commentsBeforeScan.size + (if (shouldAddComment) 1 else 0)) - issueFun(repoPR.currentPR()) - } - } - } - - def waitUntil[T](shouldAddComment: Boolean)(issueFun: PullRequest => T)(implicit repoPR: RepoPR) { + def waitUntil[T](shouldAddComment: Boolean)(issueFun: PullRequest => T)(implicit repoPR: RepoPR): Unit = { val commentsBeforeScan = repoPR.listComments() eventually { val commentsAfterScan = repoPR.listComments() @@ -116,11 +126,11 @@ trait Helpers extends PlaySpec with OneAppPerSuite with Inspectors with ScalaFut } } - def scanShouldNotChangeAnything[T,S]()(implicit meat: RepoPR) { + def scanShouldNotChangeAnything()(implicit meat: RepoPR): Unit = { scanShouldNotChange { pr => (pr.labels.list().all().futureValue, pr.comments) } } - def scanShouldNotChange[T,S](issueState: PullRequest => S)(implicit repoPR: RepoPR) { + def scanShouldNotChange[S](issueState: PullRequest => S)(implicit repoPR: RepoPR): Unit = { val issueBeforeScan = repoPR.currentPR() val beforeState = issueState(issueBeforeScan) diff --git a/test/lib/TestRepoCreation.scala b/test/lib/TestRepoCreation.scala index 1e3979c..3c86760 100644 --- a/test/lib/TestRepoCreation.scala +++ b/test/lib/TestRepoCreation.scala @@ -1,20 +1,19 @@ package lib -import java.time.Duration.ofMinutes - -import com.google.common.io.Files.createTempDir import com.madgag.git._ -import com.madgag.scalagithub.GitHub._ import com.madgag.scalagithub.commands.CreateRepo import com.madgag.scalagithub.model.Repo import com.madgag.time.Implicits._ +import lib.gitgithub._ import org.eclipse.jgit.api.Git import org.eclipse.jgit.transport.RemoteRefUpdate import org.scalatest.BeforeAndAfterAll -import scala.collection.convert.wrapAll._ +import java.nio.file.Files.createTempDirectory +import java.time.Duration.ofMinutes import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future +import scala.jdk.CollectionConverters._ trait TestRepoCreation extends Helpers with BeforeAndAfterAll { @@ -23,7 +22,7 @@ trait TestRepoCreation extends Helpers with BeforeAndAfterAll { def isTestRepo(repo: Repo) = repo.name.startsWith(testRepoNamePrefix) && repo.created_at.toInstant.age() > ofMinutes(10) - override def beforeAll { + override def beforeAll(): Unit = { val oldRepos = github.listRepos("updated", "desc").all().futureValue.filter(isTestRepo) Future.traverse(oldRepos)(_.delete()) } @@ -49,10 +48,11 @@ trait TestRepoCreation extends Helpers with BeforeAndAfterAll { localGitRepo.git.branchCreate().setName(defaultBranchName).setStartPoint("HEAD").call() } - val pushResults = localGitRepo.git.push.setCredentialsProvider(Bot.githubCredentials.git).setPushTags().setPushAll().call() + val pushResults = + localGitRepo.git.push.setCredentialsProvider(githubCredentials.git).setPushTags().setPushAll().call() - forAll (pushResults.toSeq) { pushResult => - all (pushResult.getRemoteUpdates.map(_.getStatus)) must be(RemoteRefUpdate.Status.OK) + forAll (pushResults.asScala) { pushResult => + all (pushResult.getRemoteUpdates.asScala.map(_.getStatus)) must be(RemoteRefUpdate.Status.OK) } eventually { @@ -60,7 +60,8 @@ trait TestRepoCreation extends Helpers with BeforeAndAfterAll { } val clonedRepo = eventually { - Git.cloneRepository().setBare(true).setURI(testGithubRepo.clone_url).setDirectory(createTempDir()).call() + Git.cloneRepository().setBare(true).setURI(testGithubRepo.clone_url) + .setDirectory(createTempDirectory("prout-test-repo").toFile).call() } require(clonedRepo.getRepository.getRef(defaultBranchName).getObjectId == localGitRepo.resolve("HEAD")) diff --git a/test/resources/logback.xml b/test/resources/logback.xml index 0fb9344..e64537f 100644 --- a/test/resources/logback.xml +++ b/test/resources/logback.xml @@ -1,6 +1,6 @@ - + logs/application-test.log