@@ -14,7 +14,7 @@ patches, applies patches on top of an existing tree, triggers appropriate
builds and test suites, and reports the results.
At present, snowpatch supports
-[patchwork-freedesktop](http://github.com/dlespiau/patchwork) and
+[Patchwork](http://jk.ozlabs.org/projects/patchwork/) and
[Jenkins](http://jenkins-ci.org).
snowpatch is named in honour of
@@ -108,10 +108,18 @@ impl JenkinsBackend {
fn get_api_json_object(&self, base_url: &str) -> Value {
// TODO: Don't panic on failure, fail more gracefully
let url = format!("{}api/json", base_url);
- let mut resp = self.get(&url).send().expect("HTTP request error");
let mut result_str = String::new();
- resp.read_to_string(&mut result_str)
- .unwrap_or_else(|err| panic!("Couldn't read from server: {}", err));
+ loop {
+ let mut resp = self.get(&url).send().expect("HTTP request error");
+
+ if resp.status.is_server_error() {
+ sleep(Duration::from_millis(JENKINS_POLLING_INTERVAL));
+ continue;
+ }
+ resp.read_to_string(&mut result_str)
+ .unwrap_or_else(|err| panic!("Couldn't read from server: {}", err));
+ break;
+ }
serde_json::from_str(&result_str).unwrap_or_else(
|err| panic!("Couldn't parse JSON from Jenkins: {}", err)
)
@@ -73,8 +73,10 @@ mod utils;
static USAGE: &'static str = "
Usage:
- snowpatch <config-file> [--count=<count> | --series <id>] [--project <name>]
+ snowpatch <config-file> [--count=<count>] [--project <name>]
snowpatch <config-file> --mbox <mbox> --project <name>
+ snowpatch <config-file> --patch <id>
+ snowpatch <config-file> --series <id>
snowpatch -v | --version
snowpatch -h | --help
@@ -82,6 +84,7 @@ By default, snowpatch runs as a long-running daemon.
Options:
--count <count> Run tests on <count> recent series.
+ --patch <id> Run tests on the given Patchwork patch.
--series <id> Run tests on the given Patchwork series.
--mbox <mbox> Run tests on the given mbox file. Requires --project
--project <name> Test patches for the given project.
@@ -93,6 +96,7 @@ Options:
struct Args {
arg_config_file: String,
flag_count: u16,
+ flag_patch: u32,
flag_series: u32,
flag_mbox: String,
flag_project: String,
@@ -135,11 +139,11 @@ fn run_tests(settings: &Config, client: Arc<Client>, project: &Project, tag: &st
let test_result = jenkins.get_build_result(&build_url_real).unwrap();
info!("Jenkins job for {}/{} complete.", branch_name, job.title);
results.push(TestResult {
- test_name: format!("Test {} on branch {}", job.title,
- branch_name.to_string()).to_string(),
+ description: Some(format!("Test {} on branch {}", job.title,
+ branch_name.to_string()).to_string()),
state: test_result,
- url: Some(jenkins.get_results_url(&build_url_real, &job.parameters)),
- summary: Some("TODO: get this summary from Jenkins".to_string()),
+ context: Some(format!("{}-{}", "snowpatch", job.title.replace("/", "_")).to_string()),
+ target_url: Some(jenkins.get_results_url(&build_url_real, &job.parameters)),
});
}
results
@@ -199,19 +203,25 @@ fn test_patch(settings: &Config, client: &Arc<Client>, project: &Project, path:
Ok(_) => {
successfully_applied = true;
results.push(TestResult {
- test_name: "apply_patch".to_string(),
state: TestState::Success,
- url: None,
- summary: Some(format!("Successfully applied to branch {}", branch_name)),
+ description: Some(format!("{}/{}\n\n{}",
+ branch_name.to_string(),
+ "apply_patch".to_string(),
+ "Successfully applied".to_string())
+ .to_string()),
+ .. Default::default()
});
},
Err(_) => {
// It didn't apply. No need to bother testing.
results.push(TestResult {
- test_name: "apply_patch".to_string(),
state: TestState::Warning,
- url: None,
- summary: Some(format!("Failed to apply to branch {}", branch_name)),
+ description: Some(format!("{}/{}\n\n{}",
+ branch_name.to_string(),
+ "apply_patch".to_string(),
+ "Patch failed to apply".to_string())
+ .to_string()),
+ .. Default::default()
});
continue;
}
@@ -234,10 +244,9 @@ fn test_patch(settings: &Config, client: &Arc<Client>, project: &Project, path:
if !successfully_applied {
results.push(TestResult {
- test_name: "apply_patch".to_string(),
state: TestState::Fail,
- url: None,
- summary: Some("Failed to apply to any branch".to_string()),
+ description: Some("Failed to apply to any branch".to_string()),
+ .. Default::default()
});
}
results
@@ -293,14 +302,15 @@ fn main() {
});
let mut patchwork = PatchworkServer::new(&settings.patchwork.url, &client);
- if settings.patchwork.user.is_some() {
- debug!("Patchwork authentication set for user {}",
- &settings.patchwork.user.clone().unwrap());
- patchwork.set_authentication(&settings.patchwork.user.clone().unwrap(),
- &settings.patchwork.pass.clone());
- }
+ patchwork.set_authentication(&settings.patchwork.user,
+ &settings.patchwork.pass,
+ &settings.patchwork.token);
let patchwork = patchwork;
+ if args.flag_series > 0 && args.flag_patch > 0 {
+ panic!("Can't specify both --series and --patch");
+ }
+
if args.flag_mbox != "" && args.flag_project != "" {
info!("snowpatch is testing a local patch.");
let patch = Path::new(&args.flag_mbox);
@@ -314,64 +324,120 @@ fn main() {
return;
}
+ if args.flag_patch > 0 {
+ info!("snowpatch is testing a patch from Patchwork.");
+ let patch = patchwork.get_patch(&(args.flag_patch as u64)).unwrap();
+ match settings.projects.get(&patch.project.link_name) {
+ None => panic!("Couldn't find project {}", &patch.project.link_name),
+ Some(project) => {
+ let mbox = if patch.has_series() {
+ let dependencies = patchwork.get_patch_dependencies(&patch);
+ patchwork.get_patches_mbox(dependencies)
+ } else {
+ patchwork.get_patch_mbox(&patch)
+ };
+ test_patch(&settings, &client, project, &mbox);
+ }
+ }
+ return;
+ }
+
if args.flag_series > 0 {
info!("snowpatch is testing a series from Patchwork.");
let series = patchwork.get_series(&(args.flag_series as u64)).unwrap();
- match settings.projects.get(&series.project.linkname) {
- None => panic!("Couldn't find project {}", &series.project.linkname),
+ // The last patch in the series, so its dependencies are the whole series
+ let patch = patchwork.get_patch_by_url(&series.patches.last().unwrap().url).unwrap();
+ // We have to do it this way since there's no project field on Series
+ let project = patchwork.get_project(&patch.project.name).unwrap();
+ match settings.projects.get(&project.link_name) {
+ None => panic!("Couldn't find project {}", &project.link_name),
Some(project) => {
- let patch = patchwork.get_patch(&series);
- test_patch(&settings, &client, project, &patch);
+ let dependencies = patchwork.get_patch_dependencies(&patch);
+ let mbox = patchwork.get_patches_mbox(dependencies);
+ test_patch(&settings, &client, project, &mbox);
}
}
-
return;
}
- // The number of series tested so far. If --count isn't provided, this is unused.
- let mut series_count = 0;
+ // The number of patches tested so far. If --count isn't provided, this is unused.
+ let mut patch_count = 0;
- // Poll patchwork for new series. For each series, get patches, apply and test.
+ /*
+ * Poll Patchwork for new patches.
+ * If the patch is standalone (not part of a series), apply it.
+ * If the patch is part of a series, apply all of its dependencies.
+ * Spawn tests.
+ */
'daemon: loop {
- let series_list = patchwork.get_series_query().unwrap().results.unwrap();
+ let patch_list = patchwork.get_patch_query().unwrap_or_else(
+ |err| panic!("Failed to obtain patch list: {}", err));
info!("snowpatch is ready to test new revisions from Patchwork.");
- for series in series_list {
+ for patch in patch_list {
// If it's already been tested, we can skip it
- if series.test_state.is_some() {
- debug!("Skipping already tested series {} ({})", series.name, series.id);
+ if patch.check != "pending" {
+ debug!("Skipping already tested patch {}", patch.name);
+ continue;
+ }
+
+ if !patch.action_required() {
+ debug!("Skipping patch {} in state {}", patch.name, patch.state);
continue;
}
+ //let project = patchwork.get_project(&patch.project).unwrap();
// Skip if we're using -p and it's the wrong project
- if args.flag_project != "" && series.project.linkname != args.flag_project {
- debug!("Skipping series {} ({}) (wrong project: {})",
- series.name, series.id, series.project.linkname);
+ if args.flag_project != "" && patch.project.link_name != args.flag_project {
+ debug!("Skipping patch {} ({}) (wrong project: {})",
+ patch.name, patch.id, patch.project.link_name);
continue;
}
- match settings.projects.get(&series.project.linkname) {
+ match settings.projects.get(&patch.project.link_name) {
None => {
- debug!("Project {} not configured for series {} ({})",
- &series.project.linkname, series.name, series.id);
+ debug!("Project {} not configured for patch {}",
+ &patch.project.link_name, patch.name);
continue;
},
Some(project) => {
- let patch = patchwork.get_patch(&series);
- let results = test_patch(&settings, &client, project, &patch);
+ // TODO(ajd): Refactor this.
+ let mbox = if patch.has_series() {
+ debug!("Patch {} has a series at {}!", &patch.name, &patch.series[0].url);
+ let series = patchwork.get_series_by_url(&patch.series[0].url);
+ match series {
+ Ok(series) => {
+ if !series.received_all {
+ debug!("Series is incomplete, skipping patch for now");
+ continue;
+ }
+ let dependencies = patchwork.get_patch_dependencies(&patch);
+ patchwork.get_patches_mbox(dependencies)
+
+ },
+ Err(e) => {
+ debug!("Series is not OK: {}", e);
+ patchwork.get_patch_mbox(&patch)
+ }
+ }
+ } else {
+ patchwork.get_patch_mbox(&patch)
+ };
+
+ let results = test_patch(&settings, &client, project, &mbox);
+
// Delete the temporary directory with the patch in it
- fs::remove_dir_all(patch.parent().unwrap()).unwrap_or_else(
+ fs::remove_dir_all(mbox.parent().unwrap()).unwrap_or_else(
|err| error!("Couldn't delete temp directory: {}", err));
if project.push_results {
for result in results {
- patchwork.post_test_result(result, &series.id,
- &series.version).unwrap();
+ patchwork.post_test_result(result, &patch.checks).unwrap();
}
}
if args.flag_count > 0 {
- series_count += 1;
- debug!("Tested {} series out of {}",
- series_count, args.flag_count);
- if series_count >= args.flag_count {
+ patch_count += 1;
+ debug!("Tested {} patches out of {}",
+ patch_count, args.flag_count);
+ if patch_count >= args.flag_count {
break 'daemon;
}
}
@@ -18,8 +18,9 @@ use std;
use std::io::{self};
use std::option::Option;
use std::path::PathBuf;
-use std::fs::File;
+use std::fs::{File, OpenOptions};
use std::result::Result;
+use std::collections::BTreeMap;
use tempdir::TempDir;
@@ -31,56 +32,130 @@ use hyper::mime::{Mime, TopLevel, SubLevel, Attr, Value};
use hyper::status::StatusCode;
use hyper::client::response::Response;
+use serde::{self, Serializer};
use serde_json;
use utils;
// TODO: more constants. constants for format strings of URLs and such.
pub static PATCHWORK_API: &'static str = "/api/1.0";
-pub static PATCHWORK_QUERY: &'static str = "?ordering=-last_updated&related=expand";
+pub static PATCHWORK_QUERY: &'static str = "?order=-id";
-// /api/1.0/projects/*/series/
+#[derive(Deserialize, Clone)]
+pub struct SubmitterSummary {
+ pub id: u64,
+ pub url: String,
+ pub name: String,
+ pub email: String
+}
+
+#[derive(Deserialize, Clone)]
+pub struct DelegateSummary {
+ pub id: u64,
+ pub url: String,
+ pub first_name: String,
+ pub last_name: String,
+ pub email: String
+}
+// /api/1.0/projects/{id}
#[derive(Deserialize, Clone)]
pub struct Project {
pub id: u64,
+ pub url: String,
pub name: String,
- pub linkname: String,
- pub listemail: String,
+ pub link_name: String,
+ pub list_email: String,
+ pub list_id: String,
pub web_url: Option<String>,
pub scm_url: Option<String>,
- pub webscm_url: Option<String>
+ pub webscm_url: Option<String>,
+}
+
+// /api/1.0/patches/
+// This omits fields from /patches/{id}, deal with it for now.
+
+#[derive(Deserialize, Clone)]
+pub struct Patch {
+ pub id: u64,
+ pub url: String,
+ pub project: Project,
+ pub msgid: String,
+ pub date: String,
+ pub name: String,
+ pub commit_ref: Option<String>,
+ pub pull_url: Option<String>,
+ pub state: String, // TODO enum of possible states
+ pub archived: bool,
+ pub hash: Option<String>,
+ pub submitter: SubmitterSummary,
+ pub delegate: Option<DelegateSummary>,
+ pub mbox: String,
+ pub series: Vec<SeriesSummary>,
+ pub check: String, // TODO enum of possible states
+ pub checks: String,
+ pub tags: BTreeMap<String, u64>
+}
+
+impl Patch {
+ pub fn has_series(&self) -> bool {
+ !&self.series.is_empty()
+ }
+
+ pub fn action_required(&self) -> bool {
+ &self.state == "new" || &self.state == "under-review"
+ }
+}
+
+#[derive(Deserialize, Clone)]
+pub struct PatchSummary {
+ pub date: String,
+ pub id: u64,
+ pub mbox: String,
+ pub msgid: String,
+ pub name: String,
+ pub url: String
}
#[derive(Deserialize, Clone)]
-pub struct Submitter {
+pub struct CoverLetter {
+ pub date: String,
pub id: u64,
- pub name: String
+ pub msgid: String,
+ pub name: String,
+ pub url: String
}
+// /api/1.0/series/
+// The series list and /series/{id} are the same, luckily
#[derive(Deserialize, Clone)]
pub struct Series {
+ pub cover_letter: Option<CoverLetter>,
+ pub date: String,
pub id: u64,
+ pub mbox: String,
+ pub name: Option<String>,
+ pub patches: Vec<PatchSummary>,
pub project: Project,
- pub name: String,
- pub n_patches: u64,
- pub submitter: Submitter,
- pub submitted: String,
- pub last_updated: String,
- pub version: u64,
- pub reviewer: Option<String>,
- pub test_state: Option<String>
+ pub received_all: bool,
+ pub received_total: u64,
+ pub submitter: SubmitterSummary,
+ pub total: u64,
+ pub url: String,
+ pub version: u64
}
-#[derive(Deserialize)]
-pub struct SeriesList {
- pub count: u64,
- pub next: Option<String>,
- pub previous: Option<String>,
- pub results: Option<Vec<Series>>
+#[derive(Deserialize, Clone)]
+pub struct SeriesSummary {
+ pub id: u64,
+ pub url: String,
+ pub date: String,
+ pub name: Option<String>,
+ pub version: u64,
+ pub mbox: String,
}
-#[derive(Serialize, Clone)]
+#[derive(Serialize, Clone, PartialEq)]
pub enum TestState {
#[serde(rename = "pending")]
Pending,
@@ -99,12 +174,39 @@ impl Default for TestState {
}
// /api/1.0/series/*/revisions/*/test-results/
-#[derive(Serialize)]
+#[derive(Serialize, Default, Clone)]
pub struct TestResult {
- pub test_name: String,
pub state: TestState,
- pub url: Option<String>,
- pub summary: Option<String>
+ #[serde(serialize_with = "TestResult::serialize_target_url")]
+ pub target_url: Option<String>,
+ pub description: Option<String>,
+ #[serde(serialize_with = "TestResult::serialize_context")]
+ pub context: Option<String>,
+}
+
+impl TestResult {
+ fn serialize_target_url<S>(target_url: &Option<String>, ser: S)
+ -> Result<S::Ok, S::Error> where S: Serializer {
+ if target_url.is_none() {
+ serde::Serialize::serialize(&Some("http://no.url".to_string()), ser)
+ } else {
+ serde::Serialize::serialize(target_url, ser)
+ }
+ }
+
+ fn serialize_context<S>(context: &Option<String>, ser: S)
+ -> Result<S::Ok, S::Error> where S: Serializer {
+ if context.is_none() {
+ serde::Serialize::serialize(
+ &Some(format!("{}-{}",
+ env!("CARGO_PKG_NAME"),
+ env!("CARGO_PKG_VERSION")).to_string()
+ .replace(".", "_")),
+ ser)
+ } else {
+ serde::Serialize::serialize(context, ser)
+ }
+ }
}
pub struct PatchworkServer {
@@ -133,14 +235,31 @@ impl PatchworkServer {
}
#[cfg_attr(feature="cargo-clippy", allow(ptr_arg))]
- pub fn set_authentication(&mut self, username: &String, password: &Option<String>) {
- self.headers.set(Authorization(Basic {
- username: username.clone(),
- password: password.clone(),
- }));
+ pub fn set_authentication(&mut self, username: &Option<String>,
+ password: &Option<String>,
+ token: &Option<String>) {
+ match (username, password, token) {
+ (&None, &None, &Some(ref token)) => {
+ self.headers.set(Authorization(
+ format!("Token {}", token)));
+ },
+ (&Some(ref username), &Some(ref password), &None) => {
+ self.headers.set(Authorization(Basic {
+ username: username.clone(),
+ password: Some(password.clone()),
+ }));
+ },
+ _ => panic!("Invalid patchwork authentication details"),
+ }
}
- fn get(&self, url: &str) -> std::result::Result<String, hyper::error::Error> {
+ pub fn get_url(&self, url: &str)
+ -> std::result::Result<Response, hyper::error::Error> {
+ self.client.get(&*url).headers(self.headers.clone())
+ .header(Connection::close()).send()
+ }
+
+ pub fn get_url_string(&self, url: &str) -> std::result::Result<String, hyper::error::Error> {
let mut resp = try!(self.client.get(&*url).headers(self.headers.clone())
.header(Connection::close()).send());
let mut body: Vec<u8> = vec![];
@@ -148,51 +267,63 @@ impl PatchworkServer {
Ok(String::from_utf8(body).unwrap())
}
- pub fn post_test_result(&self, result: TestResult,
- series_id: &u64, series_revision: &u64)
+ pub fn post_test_result(&self, result: TestResult, checks_url: &str)
-> Result<StatusCode, hyper::error::Error> {
let encoded = serde_json::to_string(&result).unwrap();
let headers = self.headers.clone();
debug!("JSON Encoded: {}", encoded);
- let res = try!(self.client.post(&format!(
- "{}{}/series/{}/revisions/{}/test-results/",
- &self.url, PATCHWORK_API, &series_id, &series_revision))
- .headers(headers).body(&encoded).send());
- assert_eq!(res.status, hyper::status::StatusCode::Created);
- Ok(res.status)
+ let mut resp = try!(self.client.post(checks_url)
+ .headers(headers).body(&encoded).send());
+ let mut body: Vec<u8> = vec![];
+ io::copy(&mut resp, &mut body).unwrap();
+ trace!("{}", String::from_utf8(body).unwrap());
+ assert_eq!(resp.status, hyper::status::StatusCode::Created);
+ Ok(resp.status)
}
- pub fn get_series(&self, series_id: &u64) -> Result<Series, serde_json::Error> {
- let url = format!("{}{}/series/{}{}", &self.url, PATCHWORK_API,
- series_id, PATCHWORK_QUERY);
- serde_json::from_str(&self.get(&url).unwrap())
+ pub fn get_project(&self, url: &str) -> Result<Project, serde_json::Error> {
+ serde_json::from_str(&self.get_url_string(url).unwrap())
}
- pub fn get_series_mbox(&self, series_id: &u64, series_revision: &u64)
- -> std::result::Result<Response, hyper::error::Error> {
- let url = format!("{}{}/series/{}/revisions/{}/mbox/",
- &self.url, PATCHWORK_API, series_id, series_revision);
- self.client.get(&*url).headers(self.headers.clone())
- .header(Connection::close()).send()
+ pub fn get_patch(&self, patch_id: &u64) -> Result<Patch, serde_json::Error> {
+ let url = format!("{}{}/patches/{}{}", &self.url, PATCHWORK_API,
+ patch_id, PATCHWORK_QUERY);
+ serde_json::from_str(&self.get_url_string(&url).unwrap())
+ }
+
+ pub fn get_patch_by_url(&self, url: &str) -> Result<Patch, serde_json::Error> {
+ serde_json::from_str(&self.get_url_string(url).unwrap())
}
- pub fn get_series_query(&self) -> Result<SeriesList, serde_json::Error> {
- let url = format!("{}{}/series/{}", &self.url,
- PATCHWORK_API, PATCHWORK_QUERY);
- serde_json::from_str(&self.get(&url).unwrap_or_else(
+ pub fn get_patch_query(&self) -> Result<Vec<Patch>, serde_json::Error> {
+ let url = format!("{}{}/patches/{}", &self.url, PATCHWORK_API, PATCHWORK_QUERY);
+ serde_json::from_str(&self.get_url_string(&url).unwrap_or_else(
|err| panic!("Failed to connect to Patchwork: {}", err)))
}
- pub fn get_patch(&self, series: &Series) -> PathBuf {
+ pub fn get_patch_dependencies(&self, patch: &Patch) -> Vec<Patch> {
+ // We assume the list of patches in a series are in order.
+ let mut dependencies: Vec<Patch> = vec!();
+ let series = self.get_series_by_url(&patch.series[0].url);
+ if series.is_err() {
+ return dependencies;
+ }
+ for dependency in series.unwrap().patches {
+ dependencies.push(self.get_patch_by_url(&dependency.url).unwrap());
+ if dependency.url == patch.url {
+ break;
+ }
+ }
+ dependencies
+ }
+
+ pub fn get_patch_mbox(&self, patch: &Patch) -> PathBuf {
let dir = TempDir::new("snowpatch").unwrap().into_path();
let mut path = dir.clone();
- let tag = utils::sanitise_path(
- format!("{}-{}-{}", series.submitter.name,
- series.id, series.version));
+ let tag = utils::sanitise_path(patch.name.clone());
path.push(format!("{}.mbox", tag));
- let mut mbox_resp = self.get_series_mbox(&series.id, &series.version)
- .unwrap();
+ let mut mbox_resp = self.get_url(&patch.mbox).unwrap();
debug!("Saving patch to file {}", path.display());
let mut mbox = File::create(&path).unwrap_or_else(
@@ -201,4 +332,37 @@ impl PatchworkServer {
|err| panic!("Couldn't save mbox from Patchwork: {}", err));
path
}
+
+ pub fn get_patches_mbox(&self, patches: Vec<Patch>) -> PathBuf {
+ let dir = TempDir::new("snowpatch").unwrap().into_path();
+ let mut path = dir.clone();
+ let tag = utils::sanitise_path(patches.last().unwrap().name.clone());
+ path.push(format!("{}.mbox", tag));
+
+ let mut mbox = OpenOptions::new()
+ .create(true)
+ .write(true)
+ .append(true)
+ .open(&path)
+ .unwrap_or_else(|err| panic!("Couldn't make file: {}", err));
+
+ for patch in patches {
+ let mut mbox_resp = self.get_url(&patch.mbox).unwrap();
+ debug!("Appending patch {} to file {}", patch.name, path.display());
+ io::copy(&mut mbox_resp, &mut mbox).unwrap_or_else(
+ |err| panic!("Couldn't save mbox from Patchwork: {}", err));
+ }
+ path
+ }
+
+ pub fn get_series(&self, series_id: &u64) -> Result<Series, serde_json::Error> {
+ let url = format!("{}{}/series/{}{}", &self.url, PATCHWORK_API,
+ series_id, PATCHWORK_QUERY);
+ serde_json::from_str(&self.get_url_string(&url).unwrap())
+ }
+
+ pub fn get_series_by_url(&self, url: &str) -> Result<Series, serde_json::Error> {
+ serde_json::from_str(&self.get_url_string(url).unwrap())
+ }
+
}
@@ -39,8 +39,10 @@ pub struct Git {
pub struct Patchwork {
pub url: String,
pub port: Option<u16>,
+ // TODO: Enforce (user, pass) XOR token
pub user: Option<String>,
pub pass: Option<String>,
+ pub token: Option<String>,
pub polling_interval: u64,
}
@@ -63,6 +65,7 @@ pub struct Project {
pub remote_uri: String,
pub jobs: Vec<Job>,
pub push_results: bool,
+ pub category: Option<String>,
}
impl Project {