diff options
30 files changed, 793 insertions, 46 deletions
diff --git a/.envrc b/.envrc new file mode 100644 index 0000000000..1d11c53545 --- /dev/null +++ b/.envrc @@ -0,0 +1,2 @@ +use nix +watch_file nix/*.nix diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index bd2abdad87..5b75623fd4 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,8 +1,8 @@ <!-- -Hey there! Thanks for your contribution for MeshMC. +Hey there! Thanks for your contribution for Project Tick. Please make sure that your commits are signed off and please sign CLA first. -If you don't know how that works, check out our contribution guidelines: https://github.com/Project-Tick/MeshMC/blob/master/CONTRIBUTING.md#signing-your-work +If you don't know how that works, check out our contribution guidelines: https://github.com/Project-Tick/Project-Tick/blob/master/CONTRIBUTING.md#signing-your-work If you already created your commits, you can run `git rebase --signoff develop` to retroactively sign-off all your commits and `git push --force` to override what you have pushed already. Note that signing and signing-off are two different things! diff --git a/.gitignore b/.gitignore index 6ac77d6f4d..ba0dc780cb 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,5 @@ node_modules/ #Ignore vscode AI rules .github/instructions/codacy.instructions.md + +tree.txt diff --git a/meshmc/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index f3f877ff21..f3f877ff21 100644 --- a/meshmc/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md diff --git a/REUSE.toml b/REUSE.toml index 5ec4afbba4..d4db8d268c 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -154,3 +154,8 @@ SPDX-FileCopyrightText = "MultiMC Contributors & PolyMC Contributors & PrismLaun path = ["images4docker/**"] SPDX-License-Identifier = "GPL-3.0-or-later" SPDX-FileCopyrightText = "Project Tick" + +[[annotations]] +path = ["ofborg/**"] +SPDX-License-Identifier = "MIT" +SPDX-FileCopyrightText = "NixOS Contributors & Project Tick" diff --git a/meshmc/bootstrap.cmd b/bootstrap.cmd index 9da8b46195..9da8b46195 100644 --- a/meshmc/bootstrap.cmd +++ b/bootstrap.cmd diff --git a/meshmc/bootstrap.sh b/bootstrap.sh index 3762deaa2e..3762deaa2e 100755 --- a/meshmc/bootstrap.sh +++ b/bootstrap.sh diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000000..55a204d6b5 --- /dev/null +++ b/flake.lock @@ -0,0 +1,24 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1775036866, + "narHash": "sha256-ByAX1LkhCwZ94+KnFAmnJSMAvui7kgCxjHgUHsWAbfI=", + "rev": "6201e203d09599479a3b3450ed24fa81537ebc4e", + "type": "tarball", + "url": "https://releases.nixos.org/nixos/unstable/nixos-26.05pre972949.6201e203d095/nixexprs.tar.xz" + }, + "original": { + "type": "tarball", + "url": "https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000000..d510656840 --- /dev/null +++ b/flake.nix @@ -0,0 +1,78 @@ +{ + description = " Project Tick is a project dedicated to providing developers with ease of use and users with long-lasting software."; + + inputs = { + nixpkgs.url = "https://channels.nixos.org/nixos-unstable/nixexprs.tar.xz"; + }; + + outputs = + { + self, + nixpkgs, + }: + + let + inherit (nixpkgs) lib; + + # While we only officially support aarch and x86_64 on Linux and MacOS, + # we expose a reasonable amount of other systems for users who want to + # build for most exotic platforms + systems = lib.systems.flakeExposed; + + forAllSystems = lib.genAttrs systems; + nixpkgsFor = forAllSystems (system: nixpkgs.legacyPackages.${system}); + in + + { + devShells = forAllSystems ( + system: + + let + pkgs = nixpkgsFor.${system}; + llvm = pkgs.llvmPackages_22; + python = pkgs.python3; + mkShell = pkgs.mkShell.override { inherit (llvm) stdenv; }; + + packages' = self.packages.${system}; + + welcomeMessage = '' + Welcome to Project Tick! + ''; + in + + { + default = mkShell { + name = "project-tick"; + + packages = [ + + (pkgs.stdenvNoCC.mkDerivation { + pname = "clang-tidy-diff"; + inherit (llvm.clang) version; + + nativeBuildInputs = [ + pkgs.installShellFiles + python.pkgs.wrapPython + ]; + + dontUnpack = true; + dontConfigure = true; + dontBuild = true; + + postInstall = "installBin ${llvm.libclang.python}/share/clang/clang-tidy-diff.py"; + postFixup = "wrapPythonPrograms"; + }) + ]; + + shellHook = '' + git submodule update --init --force + + echo ${lib.escapeShellArg welcomeMessage} + ''; + }; + } + ); + + formatter = forAllSystems (system: nixpkgsFor.${system}.nixfmt-rfc-style); + }; +} diff --git a/meshmc/lefthook.yml b/lefthook.yml index f92640489b..f92640489b 100644 --- a/meshmc/lefthook.yml +++ b/lefthook.yml diff --git a/meshmc/BUILD.md b/meshmc/BUILD.md index d9058f7eb9..118324d2f8 100644 --- a/meshmc/BUILD.md +++ b/meshmc/BUILD.md @@ -36,7 +36,7 @@ and sets up lefthook git hooks. ### Linux / macOS ```bash -./bootstrap.sh +../bootstrap.sh ``` Supported distributions: Debian, Ubuntu, Fedora, RHEL/CentOS, openSUSE, Arch Linux, macOS (via Homebrew). @@ -44,7 +44,7 @@ Supported distributions: Debian, Ubuntu, Fedora, RHEL/CentOS, openSUSE, Arch Lin ### Windows ```cmd -bootstrap.cmd +..\bootstrap.cmd ``` Uses [Scoop](https://scoop.sh) for CLI tools and [vcpkg](https://github.com/microsoft/vcpkg) for C/C++ libraries. diff --git a/meshmc/README.md b/meshmc/README.md index 87d86cc82c..b8ab5172e2 100644 --- a/meshmc/README.md +++ b/meshmc/README.md @@ -1,6 +1,6 @@ MeshMC ====== -[](https://api.reuse.software/info/github.com/Project-Tick/MeshMC) [](https://cla-assistant.io/Project-Tick/MeshMC) [](https://www.gnu.org/licenses/gpl-3.0.html) [](https://crowdin.com/project/projtlauncher) +[](https://www.gnu.org/licenses/gpl-3.0.html) [](https://crowdin.com/project/projtlauncher) MeshMC is a custom launcher for Minecraft that focuses on predictability, long term stability and simplicity. diff --git a/ofborg/.env.example b/ofborg/.env.example new file mode 100644 index 0000000000..81b01b4b68 --- /dev/null +++ b/ofborg/.env.example @@ -0,0 +1,4 @@ +# Aşağıdakileri doldur +GITHUB_APP_ID= +GITHUB_WEBHOOK_SECRET= +RABBITMQ_PASSWORD=changeme diff --git a/ofborg/config.production.json b/ofborg/config.production.json new file mode 100644 index 0000000000..3dc45cd943 --- /dev/null +++ b/ofborg/config.production.json @@ -0,0 +1,101 @@ +{ + "runner": { + "identity": "mail-tickborg-1", + "repos": ["project-tick/Project-Tick"], + "trusted_users": ["AhmetSamet06"] + }, + "checkout": { + "root": "/var/lib/tickborg/checkout" + }, + "build": { + "system": "x86_64-linux", + "build_timeout_seconds": 1800 + }, + "github_app": { + "app_id": 0, + "private_key": "/etc/tickborg/github-private-key.pem", + "oauth_client_id": "GITHUB_APP_CLIENT_ID", + "oauth_client_secret_file": "/etc/tickborg/github-oauth-secret" + }, + "github_webhook_receiver": { + "listen": "0.0.0.0:9899", + "webhook_secret_file": "/etc/tickborg/webhook-secret", + "rabbitmq": { + "ssl": false, + "host": "localhost", + "virtualhost": "/", + "username": "tickborg", + "password_file": "/etc/tickborg/rabbitmq-password" + } + }, + "evaluation_filter": { + "rabbitmq": { + "ssl": false, + "host": "localhost", + "virtualhost": "/", + "username": "tickborg", + "password_file": "/etc/tickborg/rabbitmq-password" + } + }, + "github_comment_filter": { + "rabbitmq": { + "ssl": false, + "host": "localhost", + "virtualhost": "/", + "username": "tickborg", + "password_file": "/etc/tickborg/rabbitmq-password" + } + }, + "github_comment_poster": { + "rabbitmq": { + "ssl": false, + "host": "localhost", + "virtualhost": "/", + "username": "tickborg", + "password_file": "/etc/tickborg/rabbitmq-password" + } + }, + "builder": { + "rabbitmq": { + "ssl": false, + "host": "localhost", + "virtualhost": "/", + "username": "tickborg", + "password_file": "/etc/tickborg/rabbitmq-password" + } + }, + "push_filter": { + "rabbitmq": { + "ssl": false, + "host": "localhost", + "virtualhost": "/", + "username": "tickborg", + "password_file": "/etc/tickborg/rabbitmq-password" + }, + "default_attrs": [] + }, + "log_message_collector": { + "rabbitmq": { + "ssl": false, + "host": "localhost", + "virtualhost": "/", + "username": "tickborg", + "password_file": "/etc/tickborg/rabbitmq-password" + }, + "logs_path": "/var/log/tickborg" + }, + "log_api_config": { + "listen": "127.0.0.1:9898", + "logs_path": "/var/log/tickborg", + "serve_root": "https://logs.tickborg.projecttick.net/logfile" + }, + "stats": { + "rabbitmq": { + "ssl": false, + "host": "localhost", + "virtualhost": "/", + "username": "tickborg", + "password_file": "/etc/tickborg/rabbitmq-password" + } + } +} diff --git a/ofborg/example.config.json b/ofborg/example.config.json index 0344d169ea..420ae5efdd 100644 --- a/ofborg/example.config.json +++ b/ofborg/example.config.json @@ -13,7 +13,7 @@ }, "rabbitmq": { "ssl": true, - "host": "events.tickborg.project-tick.net", + "host": "events.tickborg.projecttick.net", "virtualhost": "tickborg", "username": "...", "password": "..." diff --git a/ofborg/tickborg/src/bin/build-faker.rs b/ofborg/tickborg/src/bin/build-faker.rs index df8fcbfa50..086e96493d 100644 --- a/ofborg/tickborg/src/bin/build-faker.rs +++ b/ofborg/tickborg/src/bin/build-faker.rs @@ -42,6 +42,7 @@ async fn main() -> Result<(), Box<dyn Error>> { logs: Some((Some("logs".to_owned()), Some(logbackrk.to_lowercase()))), statusreport: Some((None, Some("scratch".to_owned()))), request_id: "bogus-request-id".to_owned(), + push: None, }; { diff --git a/ofborg/tickborg/src/bin/github-webhook-receiver.rs b/ofborg/tickborg/src/bin/github-webhook-receiver.rs index 910cd4b350..60698a5019 100644 --- a/ofborg/tickborg/src/bin/github-webhook-receiver.rs +++ b/ofborg/tickborg/src/bin/github-webhook-receiver.rs @@ -87,6 +87,24 @@ async fn setup_amqp(chan: &mut Channel) -> Result<(), Box<dyn Error + Send + Syn no_wait: false, }) .await?; + + let queue_name = String::from("push-build-inputs"); + chan.declare_queue(easyamqp::QueueConfig { + queue: queue_name.clone(), + passive: false, + durable: true, + exclusive: false, + auto_delete: false, + no_wait: false, + }) + .await?; + chan.bind_queue(easyamqp::BindQueueConfig { + queue: queue_name.clone(), + exchange: "github-events".to_owned(), + routing_key: Some(String::from("push.*")), + no_wait: false, + }) + .await?; Ok(()) } diff --git a/ofborg/tickborg/src/bin/push-filter.rs b/ofborg/tickborg/src/bin/push-filter.rs new file mode 100644 index 0000000000..81d1597e0f --- /dev/null +++ b/ofborg/tickborg/src/bin/push-filter.rs @@ -0,0 +1,105 @@ +use std::env; +use std::error::Error; + +use tracing::{error, info}; + +use tickborg::config; +use tickborg::easyamqp::{self, ChannelExt, ConsumerExt}; +use tickborg::easylapin; +use tickborg::tasks; + +#[tokio::main] +async fn main() -> Result<(), Box<dyn Error>> { + tickborg::setup_log(); + + let arg = env::args() + .nth(1) + .unwrap_or_else(|| panic!("usage: {} <config>", std::env::args().next().unwrap())); + let cfg = config::load(arg.as_ref()); + + let Some(filter_cfg) = config::load(arg.as_ref()).push_filter else { + error!("No push filter configuration found!"); + panic!(); + }; + + let conn = easylapin::from_config(&filter_cfg.rabbitmq).await?; + let mut chan = conn.create_channel().await?; + + chan.declare_exchange(easyamqp::ExchangeConfig { + exchange: "github-events".to_owned(), + exchange_type: easyamqp::ExchangeType::Topic, + passive: false, + durable: true, + auto_delete: false, + no_wait: false, + internal: false, + }) + .await?; + + // Declare the build-jobs exchange (fanout) that the builder consumes from + chan.declare_exchange(easyamqp::ExchangeConfig { + exchange: "build-jobs".to_owned(), + exchange_type: easyamqp::ExchangeType::Fanout, + passive: false, + durable: true, + auto_delete: false, + no_wait: false, + internal: false, + }) + .await?; + + // Declare the build-results exchange for the comment poster + chan.declare_exchange(easyamqp::ExchangeConfig { + exchange: "build-results".to_owned(), + exchange_type: easyamqp::ExchangeType::Fanout, + passive: false, + durable: true, + auto_delete: false, + no_wait: false, + internal: false, + }) + .await?; + + let queue_name = String::from("push-build-inputs"); + chan.declare_queue(easyamqp::QueueConfig { + queue: queue_name.clone(), + passive: false, + durable: true, + exclusive: false, + auto_delete: false, + no_wait: false, + }) + .await?; + + chan.bind_queue(easyamqp::BindQueueConfig { + queue: queue_name.clone(), + exchange: "github-events".to_owned(), + routing_key: Some("push.*".to_owned()), + no_wait: false, + }) + .await?; + + let handle = easylapin::WorkerChannel(chan) + .consume( + tasks::pushfilter::PushFilterWorker::new( + cfg.acl(), + filter_cfg.default_attrs, + ), + easyamqp::ConsumeConfig { + queue: queue_name.clone(), + consumer_tag: format!("{}-push-filter", cfg.whoami()), + no_local: false, + no_ack: false, + no_wait: false, + exclusive: false, + }, + ) + .await?; + + info!("Fetching push events from {}", &queue_name); + handle.await; + + drop(conn); // Close connection. + info!("Closed the session... EOF"); + Ok(()) +} diff --git a/ofborg/tickborg/src/config.rs b/ofborg/tickborg/src/config.rs index 7d7475e3b6..623b9f5e9a 100644 --- a/ofborg/tickborg/src/config.rs +++ b/ofborg/tickborg/src/config.rs @@ -30,6 +30,8 @@ pub struct Config { pub mass_rebuilder: Option<MassRebuilder>, /// Configuration for the builder pub builder: Option<Builder>, + /// Configuration for the push filter + pub push_filter: Option<PushFilter>, /// Configuration for the log message collector pub log_message_collector: Option<LogMessageCollector>, /// Configuration for the stats server @@ -57,7 +59,7 @@ fn default_logs_path() -> String { } fn default_serve_root() -> String { - "https://logs.tickborg.project-tick.net/logfile".into() + "https://logs.tickborg.projecttick.net/logfile".into() } /// Configuration for logapi @@ -112,6 +114,18 @@ pub struct Builder { pub rabbitmq: RabbitMqConfig, } +/// Configuration for the push filter +#[derive(serde::Serialize, serde::Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct PushFilter { + /// RabbitMQ broker to connect to + pub rabbitmq: RabbitMqConfig, + /// Default projects/attrs to build when no changed projects are detected. + /// If empty and no projects detected, push builds are skipped. + #[serde(default)] + pub default_attrs: Vec<String>, +} + /// Configuration for the log message collector #[derive(serde::Serialize, serde::Deserialize, Debug)] #[serde(deny_unknown_fields)] diff --git a/ofborg/tickborg/src/ghevent/mod.rs b/ofborg/tickborg/src/ghevent/mod.rs index 243758800a..06d67acef6 100644 --- a/ofborg/tickborg/src/ghevent/mod.rs +++ b/ofborg/tickborg/src/ghevent/mod.rs @@ -1,9 +1,11 @@ mod common; mod issuecomment; mod pullrequestevent; +mod pushevent; pub use self::common::{Comment, GenericWebhook, Issue, Repository, User}; pub use self::issuecomment::{IssueComment, IssueCommentAction}; pub use self::pullrequestevent::{ PullRequest, PullRequestAction, PullRequestEvent, PullRequestState, }; +pub use self::pushevent::{HeadCommit, PushEvent, Pusher}; diff --git a/ofborg/tickborg/src/ghevent/pushevent.rs b/ofborg/tickborg/src/ghevent/pushevent.rs new file mode 100644 index 0000000000..9ae45ad9ae --- /dev/null +++ b/ofborg/tickborg/src/ghevent/pushevent.rs @@ -0,0 +1,53 @@ +use crate::ghevent::Repository; + +#[derive(serde::Serialize, serde::Deserialize, Debug)] +pub struct PushEvent { + #[serde(rename = "ref")] + pub git_ref: String, + pub before: String, + pub after: String, + pub created: bool, + pub deleted: bool, + pub forced: bool, + pub repository: Repository, + pub pusher: Pusher, + pub head_commit: Option<HeadCommit>, +} + +#[derive(serde::Serialize, serde::Deserialize, Debug)] +pub struct Pusher { + pub name: String, + pub email: Option<String>, +} + +#[derive(serde::Serialize, serde::Deserialize, Debug)] +pub struct HeadCommit { + pub id: String, + pub message: String, + pub timestamp: String, + pub added: Option<Vec<String>>, + pub removed: Option<Vec<String>>, + pub modified: Option<Vec<String>>, +} + +impl PushEvent { + /// Branch adını döndürür (refs/heads/main -> main) + pub fn branch(&self) -> Option<&str> { + self.git_ref.strip_prefix("refs/heads/") + } + + /// Tag push mi? + pub fn is_tag(&self) -> bool { + self.git_ref.starts_with("refs/tags/") + } + + /// Branch silme event'i mi? + pub fn is_delete(&self) -> bool { + self.deleted + } + + /// Boş commit (000...) mi? + pub fn is_zero_sha(&self) -> bool { + self.after.chars().all(|c| c == '0') + } +} diff --git a/ofborg/tickborg/src/message/buildjob.rs b/ofborg/tickborg/src/message/buildjob.rs index b09eae58bf..c4cc61d1fa 100644 --- a/ofborg/tickborg/src/message/buildjob.rs +++ b/ofborg/tickborg/src/message/buildjob.rs @@ -1,5 +1,5 @@ use crate::commentparser::Subset; -use crate::message::{Pr, Repo}; +use crate::message::{Pr, PushTrigger, Repo}; #[derive(serde::Serialize, serde::Deserialize, Debug)] pub struct BuildJob { @@ -10,6 +10,9 @@ pub struct BuildJob { pub request_id: String, pub logs: Option<ExchangeQueue>, // (Exchange, Routing Key) pub statusreport: Option<ExchangeQueue>, // (Exchange, Routing Key) + /// If set, this build was triggered by a push event, not a PR. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub push: Option<PushTrigger>, } #[derive(serde::Serialize, serde::Deserialize, Debug)] @@ -42,8 +45,47 @@ impl BuildJob { logs: Some(logs.unwrap_or((Some("logs".to_owned()), Some(logbackrk)))), statusreport: Some(statusreport.unwrap_or((Some("build-results".to_owned()), None))), request_id, + push: None, } } + + /// Create a build job triggered by a push event. + pub fn new_push( + repo: Repo, + push: PushTrigger, + attrs: Vec<String>, + request_id: String, + ) -> BuildJob { + let logbackrk = format!( + "{}.push.{}", + repo.full_name.to_lowercase(), + push.branch.replace('/', "-") + ); + + // Fill pr with push info so downstream consumers (comment poster, etc.) + // can still use pr.head_sha for commit statuses / check runs. + let pr = Pr { + number: 0, + head_sha: push.head_sha.clone(), + target_branch: Some(push.branch.clone()), + }; + + BuildJob { + repo, + pr, + subset: None, + attrs, + logs: Some((Some("logs".to_owned()), Some(logbackrk))), + statusreport: Some((Some("build-results".to_owned()), None)), + request_id, + push: Some(push), + } + } + + /// Returns true if this build was triggered by a push event. + pub fn is_push(&self) -> bool { + self.push.is_some() + } } pub fn from(data: &[u8]) -> Result<BuildJob, serde_json::error::Error> { diff --git a/ofborg/tickborg/src/message/buildresult.rs b/ofborg/tickborg/src/message/buildresult.rs index 122edacae3..de482f64da 100644 --- a/ofborg/tickborg/src/message/buildresult.rs +++ b/ofborg/tickborg/src/message/buildresult.rs @@ -1,4 +1,4 @@ -use crate::message::{Pr, Repo}; +use crate::message::{Pr, PushTrigger, Repo}; use hubcaps::checks::Conclusion; @@ -48,6 +48,7 @@ pub struct LegacyBuildResult { pub status: BuildStatus, pub skipped_attrs: Option<Vec<String>>, pub attempted_attrs: Option<Vec<String>>, + pub push: Option<PushTrigger>, } #[derive(serde::Serialize, serde::Deserialize, Debug)] @@ -70,6 +71,8 @@ pub enum BuildResult { status: BuildStatus, skipped_attrs: Option<Vec<String>>, attempted_attrs: Option<Vec<String>>, + #[serde(default, skip_serializing_if = "Option::is_none")] + push: Option<PushTrigger>, }, Legacy { repo: Repo, @@ -82,6 +85,8 @@ pub enum BuildResult { status: Option<BuildStatus>, skipped_attrs: Option<Vec<String>>, attempted_attrs: Option<Vec<String>>, + #[serde(default, skip_serializing_if = "Option::is_none")] + push: Option<PushTrigger>, }, } @@ -100,6 +105,7 @@ impl BuildResult { ref request_id, ref attempted_attrs, ref skipped_attrs, + ref push, .. } => LegacyBuildResult { repo: repo.to_owned(), @@ -111,6 +117,7 @@ impl BuildResult { status: self.status(), attempted_attrs: attempted_attrs.to_owned(), skipped_attrs: skipped_attrs.to_owned(), + push: push.to_owned(), }, BuildResult::V1 { ref repo, @@ -121,6 +128,7 @@ impl BuildResult { ref request_id, ref attempted_attrs, ref skipped_attrs, + ref push, .. } => LegacyBuildResult { repo: repo.to_owned(), @@ -132,6 +140,7 @@ impl BuildResult { status: self.status(), attempted_attrs: attempted_attrs.to_owned(), skipped_attrs: skipped_attrs.to_owned(), + push: push.to_owned(), }, } } @@ -143,6 +152,17 @@ impl BuildResult { } } + pub fn push(&self) -> Option<PushTrigger> { + match self { + BuildResult::Legacy { push, .. } => push.to_owned(), + BuildResult::V1 { push, .. } => push.to_owned(), + } + } + + pub fn is_push(&self) -> bool { + self.push().is_some() + } + pub fn status(&self) -> BuildStatus { match *self { BuildResult::Legacy { diff --git a/ofborg/tickborg/src/message/common.rs b/ofborg/tickborg/src/message/common.rs index c8fcd16ea2..75b5dec04f 100644 --- a/ofborg/tickborg/src/message/common.rs +++ b/ofborg/tickborg/src/message/common.rs @@ -6,9 +6,18 @@ pub struct Repo { pub clone_url: String, } -#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)] +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq)] pub struct Pr { pub target_branch: Option<String>, pub number: u64, pub head_sha: String, } + +/// Information about a push event trigger (direct push to a branch). +#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct PushTrigger { + pub head_sha: String, + pub branch: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub before_sha: Option<String>, +} diff --git a/ofborg/tickborg/src/message/mod.rs b/ofborg/tickborg/src/message/mod.rs index 03551cd1ce..8621f45668 100644 --- a/ofborg/tickborg/src/message/mod.rs +++ b/ofborg/tickborg/src/message/mod.rs @@ -4,4 +4,4 @@ pub mod buildresult; mod common; pub mod evaluationjob; -pub use self::common::{Pr, Repo}; +pub use self::common::{Pr, PushTrigger, Repo}; diff --git a/ofborg/tickborg/src/tasks/build.rs b/ofborg/tickborg/src/tasks/build.rs index 56583b28b4..2bac6d749b 100644 --- a/ofborg/tickborg/src/tasks/build.rs +++ b/ofborg/tickborg/src/tasks/build.rs @@ -121,6 +121,7 @@ impl JobActions { attempted_attrs: None, skipped_attrs: None, status: BuildStatus::Failure, + push: self.job.push.clone(), }; let result_exchange = self.result_exchange.clone(); @@ -209,6 +210,7 @@ impl JobActions { skipped_attrs: Some(not_attempted_attrs), attempted_attrs: None, status: BuildStatus::Skipped, + push: self.job.push.clone(), }; let result_exchange = self.result_exchange.clone(); @@ -249,6 +251,7 @@ impl JobActions { status, attempted_attrs: Some(attempted_attrs), skipped_attrs: Some(not_attempted_attrs), + push: self.job.push.clone(), }; let result_exchange = self.result_exchange.clone(); @@ -301,7 +304,12 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker { dyn notifyworker::NotificationReceiver + std::marker::Send + std::marker::Sync, >, ) { - let span = debug_span!("job", pr = ?job.pr.number); + let is_push = job.is_push(); + let span = if is_push { + debug_span!("job", push_branch = ?job.push.as_ref().map(|p| &p.branch), sha = %job.pr.head_sha) + } else { + debug_span!("job", pr = ?job.pr.number) + }; let _enter = span.enter(); let actions = self.actions(job, notifier); @@ -312,10 +320,19 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker { return; } - info!( - "Working on https://github.com/{}/pull/{}", - actions.job.repo.full_name, actions.job.pr.number - ); + if is_push { + let push = actions.job.push.as_ref().unwrap(); + info!( + "Working on push to {}:{} ({})", + actions.job.repo.full_name, push.branch, push.head_sha + ); + } else { + info!( + "Working on https://github.com/{}/pull/{}", + actions.job.repo.full_name, actions.job.pr.number + ); + } + let project = self.cloner.project( &actions.job.repo.full_name, actions.job.repo.clone_url.clone(), @@ -331,22 +348,38 @@ impl notifyworker::SimpleNotifyWorker for BuildWorker { let refpath = co.checkout_origin_ref(target_branch.as_ref()).unwrap(); - if co.fetch_pr(actions.job.pr.number).is_err() { - info!("Failed to fetch {}", actions.job.pr.number); - actions.pr_head_missing().await; - return; - } + if is_push { + // For push builds: the commit is already on the branch, just verify it exists + if !co.commit_exists(actions.job.pr.head_sha.as_ref()) { + info!("Push commit {} doesn't exist after fetch", actions.job.pr.head_sha); + actions.commit_missing().await; + return; + } + // Checkout the exact pushed commit + if co.checkout_ref(actions.job.pr.head_sha.as_ref()).is_err() { + info!("Failed to checkout push commit {}", actions.job.pr.head_sha); + actions.merge_failed().await; + return; + } + } else { + // For PR builds: fetch PR ref, verify commit, merge + if co.fetch_pr(actions.job.pr.number).is_err() { + info!("Failed to fetch {}", actions.job.pr.number); + actions.pr_head_missing().await; + return; + } - if !co.commit_exists(actions.job.pr.head_sha.as_ref()) { - info!("Commit {} doesn't exist", actions.job.pr.head_sha); - actions.commit_missing().await; - return; - } + if !co.commit_exists(actions.job.pr.head_sha.as_ref()) { + info!("Commit {} doesn't exist", actions.job.pr.head_sha); + actions.commit_missing().await; + return; + } - if co.merge_commit(actions.job.pr.head_sha.as_ref()).is_err() { - info!("Failed to merge {}", actions.job.pr.head_sha); - actions.merge_failed().await; - return; + if co.merge_commit(actions.job.pr.head_sha.as_ref()).is_err() { + info!("Failed to merge {}", actions.job.pr.head_sha); + actions.merge_failed().await; + return; + } } // Determine which projects to build from the requested attrs @@ -528,6 +561,7 @@ mod tests { logs: Some((Some(String::from("logs")), Some(String::from("build.log")))), statusreport: Some((Some(String::from("build-results")), None)), request_id: "bogus-request-id".to_owned(), + push: None, }; let dummyreceiver = Arc::new(notifyworker::DummyNotificationReceiver::new()); @@ -574,6 +608,7 @@ mod tests { logs: Some((Some(String::from("logs")), Some(String::from("build.log")))), statusreport: Some((Some(String::from("build-results")), None)), request_id: "bogus-request-id".to_owned(), + push: None, }; let dummyreceiver = Arc::new(notifyworker::DummyNotificationReceiver::new()); diff --git a/ofborg/tickborg/src/tasks/githubcommentposter.rs b/ofborg/tickborg/src/tasks/githubcommentposter.rs index 70c4a118e4..2f49d7401b 100644 --- a/ofborg/tickborg/src/tasks/githubcommentposter.rs +++ b/ofborg/tickborg/src/tasks/githubcommentposter.rs @@ -71,7 +71,11 @@ impl worker::SimpleWorker for GitHubCommentPoster { } }; - let span = debug_span!("job", pr = ?pr.number); + let span = if pr.number == 0 { + debug_span!("job", push_sha = %pr.head_sha) + } else { + debug_span!("job", pr = ?pr.number) + }; let _enter = span.enter(); for check in checks { @@ -111,17 +115,40 @@ fn job_to_check(job: &BuildJob, architecture: &str, timestamp: DateTime<Utc>) -> all_attrs = vec![String::from("(unknown attributes)")]; } + let details_key = if job.is_push() { + format!( + "push.{}", + job.push + .as_ref() + .map(|p| p.branch.replace('/', "-")) + .unwrap_or_default() + ) + } else { + format!("{}", job.pr.number) + }; + + let name = if job.is_push() { + let branch = job + .push + .as_ref() + .map(|p| p.branch.as_str()) + .unwrap_or("unknown"); + format!("{} on {architecture} (push to {branch})", all_attrs.join(", ")) + } else { + format!("{} on {architecture}", all_attrs.join(", ")) + }; + CheckRunOptions { - name: format!("{} on {architecture}", all_attrs.join(", ")), + name, actions: None, completed_at: None, started_at: Some(timestamp.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)), conclusion: None, details_url: Some(format!( - "https://logs.tickborg.project-tick.net/?key={}/{}.{}", + "https://logs.tickborg.projecttick.net/?key={}/{}.{}", &job.repo.owner.to_lowercase(), &job.repo.name.to_lowercase(), - job.pr.number, + details_key, )), external_id: None, head_sha: job.pr.head_sha.clone(), @@ -180,17 +207,47 @@ fn result_to_check(result: &LegacyBuildResult, timestamp: DateTime<Utc>) -> Chec String::from("No partial log is available.") }; + let is_push = result.push.is_some(); + + let details_key = if is_push { + format!( + "push.{}", + result + .push + .as_ref() + .map(|p| p.branch.replace('/', "-")) + .unwrap_or_default() + ) + } else { + format!("{}", result.pr.number) + }; + + let name = if is_push { + let branch = result + .push + .as_ref() + .map(|p| p.branch.as_str()) + .unwrap_or("unknown"); + format!( + "{} on {} (push to {branch})", + all_attrs.join(", "), + result.system + ) + } else { + format!("{} on {}", all_attrs.join(", "), result.system) + }; + CheckRunOptions { - name: format!("{} on {}", all_attrs.join(", "), result.system), + name, actions: None, completed_at: Some(timestamp.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)), started_at: None, conclusion: Some(conclusion), details_url: Some(format!( - "https://logs.tickborg.project-tick.net/?key={}/{}.{}&attempt_id={}", + "https://logs.tickborg.projecttick.net/?key={}/{}.{}&attempt_id={}", &result.repo.owner.to_lowercase(), &result.repo.name.to_lowercase(), - result.pr.number, + details_key, result.attempt_id, )), external_id: Some(result.attempt_id.clone()), @@ -244,6 +301,7 @@ mod tests { request_id: "bogus-request-id".to_owned(), attrs: vec!["foo".to_owned(), "bar".to_owned()], + push: None, }; let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); @@ -256,7 +314,7 @@ mod tests { completed_at: None, status: Some(CheckRunState::Queued), conclusion: None, - details_url: Some("https://logs.tickborg.project-tick.net/?key=project-tick/Project-Tick.2345".to_string()), + details_url: Some("https://logs.tickborg.projecttick.net/?key=project-tick/Project-Tick.2345".to_string()), external_id: None, head_sha: "abc123".to_string(), output: None, @@ -296,6 +354,7 @@ mod tests { attempted_attrs: Some(vec!["foo".to_owned()]), skipped_attrs: Some(vec!["bar".to_owned()]), status: BuildStatus::Success, + push: None, }; let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); @@ -310,7 +369,7 @@ mod tests { status: Some(CheckRunState::Completed), conclusion: Some(Conclusion::Success), details_url: Some( - "https://logs.tickborg.project-tick.net/?key=project-tick/Project-Tick.2345&attempt_id=neatattemptid" + "https://logs.tickborg.projecttick.net/?key=project-tick/Project-Tick.2345&attempt_id=neatattemptid" .to_string() ), external_id: Some("neatattemptid".to_string()), @@ -378,6 +437,7 @@ patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29 attempted_attrs: Some(vec!["foo".to_owned()]), skipped_attrs: None, status: BuildStatus::Failure, + push: None, }; let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); @@ -392,7 +452,7 @@ patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29 status: Some(CheckRunState::Completed), conclusion: Some(Conclusion::Neutral), details_url: Some( - "https://logs.tickborg.project-tick.net/?key=project-tick/Project-Tick.2345&attempt_id=neatattemptid" + "https://logs.tickborg.projecttick.net/?key=project-tick/Project-Tick.2345&attempt_id=neatattemptid" .to_string() ), external_id: Some("neatattemptid".to_string()), @@ -457,6 +517,7 @@ patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29 attempted_attrs: Some(vec!["foo".to_owned()]), skipped_attrs: None, status: BuildStatus::TimedOut, + push: None, }; let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); @@ -471,7 +532,7 @@ patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29 status: Some(CheckRunState::Completed), conclusion: Some(Conclusion::Neutral), details_url: Some( - "https://logs.tickborg.project-tick.net/?key=project-tick/Project-Tick.2345&attempt_id=neatattemptid" + "https://logs.tickborg.projecttick.net/?key=project-tick/Project-Tick.2345&attempt_id=neatattemptid" .to_string() ), external_id: Some("neatattemptid".to_string()), @@ -537,6 +598,7 @@ error: build of '/nix/store/l1limh50lx2cx45yb2gqpv7k8xl1mik2-gdb-8.1.drv' failed attempted_attrs: None, skipped_attrs: None, status: BuildStatus::Success, + push: None, }; let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); @@ -551,7 +613,7 @@ error: build of '/nix/store/l1limh50lx2cx45yb2gqpv7k8xl1mik2-gdb-8.1.drv' failed status: Some(CheckRunState::Completed), conclusion: Some(Conclusion::Success), details_url: Some( - "https://logs.tickborg.project-tick.net/?key=project-tick/Project-Tick.2345&attempt_id=neatattemptid" + "https://logs.tickborg.projecttick.net/?key=project-tick/Project-Tick.2345&attempt_id=neatattemptid" .to_string() ), external_id: Some("neatattemptid".to_string()), @@ -615,6 +677,7 @@ patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29 attempted_attrs: None, skipped_attrs: None, status: BuildStatus::Failure, + push: None, }; let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); @@ -629,7 +692,7 @@ patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29 status: Some(CheckRunState::Completed), conclusion: Some(Conclusion::Neutral), details_url: Some( - "https://logs.tickborg.project-tick.net/?key=project-tick/Project-Tick.2345&attempt_id=neatattemptid" + "https://logs.tickborg.projecttick.net/?key=project-tick/Project-Tick.2345&attempt_id=neatattemptid" .to_string() ), external_id: Some("neatattemptid".to_string()), @@ -682,6 +745,7 @@ patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29 attempted_attrs: None, skipped_attrs: Some(vec!["not-attempted".to_owned()]), status: BuildStatus::Skipped, + push: None, }; let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); @@ -695,7 +759,7 @@ patching script interpreter paths in /nix/store/pcja75y9isdvgz5i00pkrpif9rxzxc29 completed_at: Some("2023-04-20T13:37:42Z".to_string()), status: Some(CheckRunState::Completed), conclusion: Some(Conclusion::Skipped), - details_url: Some("https://logs.tickborg.project-tick.net/?key=project-tick/Project-Tick.2345&attempt_id=neatattemptid".to_string()), + details_url: Some("https://logs.tickborg.projecttick.net/?key=project-tick/Project-Tick.2345&attempt_id=neatattemptid".to_string()), external_id: Some("neatattemptid".to_string()), head_sha: "abc123".to_string(), output: Some(Output { @@ -735,6 +799,7 @@ foo attempted_attrs: None, skipped_attrs: Some(vec!["not-attempted".to_owned()]), status: BuildStatus::Skipped, + push: None, }; let timestamp = Utc.with_ymd_and_hms(2023, 4, 20, 13, 37, 42).unwrap(); @@ -748,7 +813,7 @@ foo completed_at: Some("2023-04-20T13:37:42Z".to_string()), status: Some(CheckRunState::Completed), conclusion: Some(Conclusion::Skipped), - details_url: Some("https://logs.tickborg.project-tick.net/?key=project-tick/Project-Tick.2345&attempt_id=neatattemptid".to_string()), + details_url: Some("https://logs.tickborg.projecttick.net/?key=project-tick/Project-Tick.2345&attempt_id=neatattemptid".to_string()), external_id: Some("neatattemptid".to_string()), head_sha: "abc123".to_string(), output: Some(Output { diff --git a/ofborg/tickborg/src/tasks/log_message_collector.rs b/ofborg/tickborg/src/tasks/log_message_collector.rs index 2d80f72f03..302445e2ff 100644 --- a/ofborg/tickborg/src/tasks/log_message_collector.rs +++ b/ofborg/tickborg/src/tasks/log_message_collector.rs @@ -215,7 +215,7 @@ impl worker::SimpleWorker for LogMessageCollector { // Make sure the log content exists by opening its handle. // This (hopefully) prevents builds that produce no output (for any reason) from - // having their logs.tickborg.project-tick.net link complaining about a 404. + // having their logs.tickborg.projecttick.net link complaining about a 404. let _ = self.handle_for(&job.from).unwrap(); } MsgType::Msg(ref message) => { @@ -448,6 +448,7 @@ mod tests { status: BuildStatus::Success, attempted_attrs: Some(vec!["foo".to_owned()]), skipped_attrs: Some(vec!["bar".to_owned()]), + push: None, })) }) .await diff --git a/ofborg/tickborg/src/tasks/mod.rs b/ofborg/tickborg/src/tasks/mod.rs index 5aab0fa631..3bf701870d 100644 --- a/ofborg/tickborg/src/tasks/mod.rs +++ b/ofborg/tickborg/src/tasks/mod.rs @@ -5,4 +5,5 @@ pub mod evaluationfilter; pub mod githubcommentfilter; pub mod githubcommentposter; pub mod log_message_collector; +pub mod pushfilter; pub mod statscollector; diff --git a/ofborg/tickborg/src/tasks/pushfilter.rs b/ofborg/tickborg/src/tasks/pushfilter.rs new file mode 100644 index 0000000000..8cf8d7a0ef --- /dev/null +++ b/ofborg/tickborg/src/tasks/pushfilter.rs @@ -0,0 +1,165 @@ +use crate::acl; +use crate::ghevent; +use crate::message::buildjob; +use crate::message::{PushTrigger, Repo}; +use crate::systems; +use crate::worker; + +use tracing::{debug_span, info}; +use uuid::Uuid; + +pub struct PushFilterWorker { + acl: acl::Acl, + /// Default projects/attrs to build when push doesn't match any known project. + default_attrs: Vec<String>, +} + +impl PushFilterWorker { + pub fn new(acl: acl::Acl, default_attrs: Vec<String>) -> PushFilterWorker { + PushFilterWorker { + acl, + default_attrs, + } + } +} + +impl worker::SimpleWorker for PushFilterWorker { + type J = ghevent::PushEvent; + + async fn msg_to_job( + &mut self, + _: &str, + _: &Option<String>, + body: &[u8], + ) -> Result<Self::J, String> { + match serde_json::from_slice(body) { + Ok(event) => Ok(event), + Err(err) => Err(format!( + "Failed to deserialize push event {err:?}: {:?}", + std::str::from_utf8(body).unwrap_or("<job not utf8>") + )), + } + } + + async fn consumer(&mut self, job: &ghevent::PushEvent) -> worker::Actions { + let branch = job.branch().unwrap_or_default(); + let span = debug_span!("push", branch = %branch, after = %job.after); + let _enter = span.enter(); + + if !self.acl.is_repo_eligible(&job.repository.full_name) { + info!("Repo not authorized ({})", job.repository.full_name); + return vec![worker::Action::Ack]; + } + + // Skip tag events + if job.is_tag() { + info!("Skipping tag push: {}", job.git_ref); + return vec![worker::Action::Ack]; + } + + // Skip branch deletion events + if job.is_delete() { + info!("Skipping branch delete: {}", job.git_ref); + return vec![worker::Action::Ack]; + } + + // Skip zero SHA (shouldn't happen for non-delete, but just in case) + if job.is_zero_sha() { + info!("Skipping zero SHA push"); + return vec![worker::Action::Ack]; + } + + let branch_name = branch.to_string(); + info!( + "Processing push to {}:{} ({})", + job.repository.full_name, branch_name, job.after + ); + + // Detect which projects changed from the push event's commit info + let changed_files: Vec<String> = if let Some(ref head) = job.head_commit { + let mut files = Vec::new(); + if let Some(ref added) = head.added { + files.extend(added.iter().cloned()); + } + if let Some(ref removed) = head.removed { + files.extend(removed.iter().cloned()); + } + if let Some(ref modified) = head.modified { + files.extend(modified.iter().cloned()); + } + files + } else { + Vec::new() + }; + + let attrs = if !changed_files.is_empty() { + let detected = crate::buildtool::detect_changed_projects(&changed_files); + if detected.is_empty() { + info!("No known projects changed in push, using defaults"); + self.default_attrs.clone() + } else { + info!("Detected changed projects: {:?}", detected); + detected + } + } else { + info!("No file change info in push event, using defaults"); + self.default_attrs.clone() + }; + + if attrs.is_empty() { + info!("No projects to build, skipping push"); + return vec![worker::Action::Ack]; + } + + let repo_msg = Repo { + clone_url: job.repository.clone_url.clone(), + full_name: job.repository.full_name.clone(), + owner: job.repository.owner.login.clone(), + name: job.repository.name.clone(), + }; + + let push_trigger = PushTrigger { + head_sha: job.after.clone(), + branch: branch_name, + before_sha: Some(job.before.clone()), + }; + + let request_id = Uuid::new_v4().to_string(); + + let build_job = buildjob::BuildJob::new_push( + repo_msg.clone(), + push_trigger, + attrs.clone(), + request_id, + ); + + // Schedule the build on all known architectures + let build_archs = systems::System::primary_systems(); + let mut response = vec![]; + + info!( + "Scheduling push build for {:?} on {:?}", + attrs, build_archs + ); + + for arch in &build_archs { + let (exchange, routingkey) = arch.as_build_destination(); + response.push(worker::publish_serde_action( + exchange, routingkey, &build_job, + )); + } + + // Also publish to build-results for the comment poster to pick up + response.push(worker::publish_serde_action( + Some("build-results".to_string()), + None, + &buildjob::QueuedBuildJobs { + job: build_job, + architectures: build_archs.iter().map(|a| a.to_string()).collect(), + }, + )); + + response.push(worker::Action::Ack); + response + } +} |