From patchwork Fri Aug 10 06:21:03 2018 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andrew Donnellan X-Patchwork-Id: 955962 Return-Path: X-Original-To: incoming@patchwork.ozlabs.org Delivered-To: patchwork-incoming@bilbo.ozlabs.org Received: from lists.ozlabs.org (lists.ozlabs.org [IPv6:2401:3900:2:1::3]) (using TLSv1.2 with cipher ADH-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by ozlabs.org (Postfix) with ESMTPS id 41mw4X3n5Dz9s7Q for ; Fri, 10 Aug 2018 16:21:32 +1000 (AEST) Authentication-Results: ozlabs.org; dmarc=none (p=none dis=none) header.from=au1.ibm.com Received: from lists.ozlabs.org (lists.ozlabs.org [IPv6:2401:3900:2:1::3]) by lists.ozlabs.org (Postfix) with ESMTP id 41mw4X2NSvzDrbj for ; Fri, 10 Aug 2018 16:21:32 +1000 (AEST) Authentication-Results: lists.ozlabs.org; dmarc=none (p=none dis=none) header.from=au1.ibm.com X-Original-To: snowpatch@lists.ozlabs.org Delivered-To: snowpatch@lists.ozlabs.org Authentication-Results: lists.ozlabs.org; spf=pass (mailfrom) smtp.mailfrom=au1.ibm.com (client-ip=148.163.156.1; helo=mx0a-001b2d01.pphosted.com; envelope-from=andrew.donnellan@au1.ibm.com; receiver=) Authentication-Results: lists.ozlabs.org; dmarc=none (p=none dis=none) header.from=au1.ibm.com Received: from mx0a-001b2d01.pphosted.com (mx0a-001b2d01.pphosted.com [148.163.156.1]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by lists.ozlabs.org (Postfix) with ESMTPS id 41mw4R1rZlzDqww for ; Fri, 10 Aug 2018 16:21:26 +1000 (AEST) Received: from pps.filterd (m0098394.ppops.net [127.0.0.1]) by mx0a-001b2d01.pphosted.com (8.16.0.22/8.16.0.22) with SMTP id w7A6J6vi074567 for ; Fri, 10 Aug 2018 02:21:24 -0400 Received: from e06smtp01.uk.ibm.com (e06smtp01.uk.ibm.com [195.75.94.97]) by mx0a-001b2d01.pphosted.com with ESMTP id 2kry9v2sq3-1 (version=TLSv1.2 cipher=AES256-GCM-SHA384 bits=256 verify=NOT) for ; Fri, 10 Aug 2018 02:21:24 -0400 Received: from localhost by e06smtp01.uk.ibm.com with IBM ESMTP SMTP Gateway: Authorized Use Only! Violators will be prosecuted for from ; Fri, 10 Aug 2018 07:21:22 +0100 Received: from b06cxnps4075.portsmouth.uk.ibm.com (9.149.109.197) by e06smtp01.uk.ibm.com (192.168.101.131) with IBM ESMTP SMTP Gateway: Authorized Use Only! Violators will be prosecuted; (version=TLSv1/SSLv3 cipher=AES256-GCM-SHA384 bits=256/256) Fri, 10 Aug 2018 07:21:19 +0100 Received: from d06av26.portsmouth.uk.ibm.com (d06av26.portsmouth.uk.ibm.com [9.149.105.62]) by b06cxnps4075.portsmouth.uk.ibm.com (8.14.9/8.14.9/NCO v10.0) with ESMTP id w7A6LIkG42402014 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-GCM-SHA384 bits=256 verify=FAIL); Fri, 10 Aug 2018 06:21:19 GMT Received: from d06av26.portsmouth.uk.ibm.com (unknown [127.0.0.1]) by IMSVA (Postfix) with ESMTP id 178D8AE058; Fri, 10 Aug 2018 09:21:10 +0100 (BST) Received: from d06av26.portsmouth.uk.ibm.com (unknown [127.0.0.1]) by IMSVA (Postfix) with ESMTP id 735E6AE053; Fri, 10 Aug 2018 09:21:09 +0100 (BST) Received: from ozlabs.au.ibm.com (unknown [9.192.253.14]) by d06av26.portsmouth.uk.ibm.com (Postfix) with ESMTP; Fri, 10 Aug 2018 09:21:09 +0100 (BST) Received: from intelligence.ozlabs.ibm.com (haven.au.ibm.com [9.192.254.114]) (using TLSv1.2 with cipher DHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by ozlabs.au.ibm.com (Postfix) with ESMTPSA id 6D746A0120; Fri, 10 Aug 2018 16:21:16 +1000 (AEST) From: Andrew Donnellan To: snowpatch@lists.ozlabs.org, ruscur@russell.cc Date: Fri, 10 Aug 2018 16:21:03 +1000 X-Mailer: git-send-email 2.11.0 X-TM-AS-GCONF: 00 x-cbid: 18081006-4275-0000-0000-000002A7BBF3 X-IBM-AV-DETECTION: SAVI=unused REMOTE=unused XFE=unused x-cbparentid: 18081006-4276-0000-0000-000037B0CC45 Message-Id: <20180810062103.11968-1-andrew.donnellan@au1.ibm.com> MIME-Version: 1.0 X-Proofpoint-Virus-Version: vendor=fsecure engine=2.50.10434:, , definitions=2018-08-10_04:, , signatures=0 X-Proofpoint-Spam-Details: rule=outbound_notspam policy=outbound score=0 priorityscore=1501 malwarescore=0 suspectscore=0 phishscore=0 bulkscore=0 spamscore=0 clxscore=1015 lowpriorityscore=0 mlxscore=0 impostorscore=0 mlxlogscore=999 adultscore=0 classifier=spam adjust=0 reason=mlx scancount=1 engine=8.0.1-1807170000 definitions=main-1808100068 Subject: [snowpatch] [PATCH] Support pagination when using --count X-BeenThere: snowpatch@lists.ozlabs.org X-Mailman-Version: 2.1.27 Precedence: list List-Id: Continuous Integration for patch-based workflows List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: snowpatch-bounces+incoming=patchwork.ozlabs.org@lists.ozlabs.org Sender: "snowpatch" Add support for pagination when using --count. We extract the Link header from the Patchwork response to find the correct URL for the next page. The pagination is potentially racy, as new patches could come in and disturb the ordering, but this is a limitation on the Patchwork side at present and is unlikely to be a significant problem. Closes: #17 ("Enable the ability to go to page 2 (and beyond) of Patchwork results") Signed-off-by: Andrew Donnellan --- src/main.rs | 29 ++++++++++++++++------------- src/patchwork.rs | 41 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 14 deletions(-) diff --git a/src/main.rs b/src/main.rs index b16bb31bb4b4..ba9e6dba1371 100644 --- a/src/main.rs +++ b/src/main.rs @@ -401,9 +401,6 @@ fn main() { return; } - // The number of patches tested so far. If --count isn't provided, this is unused. - let mut patch_count = 0; - /* * Poll Patchwork for new patches. * If the patch is standalone (not part of a series), apply it. @@ -411,9 +408,19 @@ fn main() { * Spawn tests. */ 'daemon: loop { - let patch_list = patchwork - .get_patch_query(&args.flag_project) - .unwrap_or_else(|err| panic!("Failed to obtain patch list: {}", err)); + let patch_list; + + // TODO: This is hacky and we should refactor out daemon vs patch/count + // mode + if args.flag_count > 0 { + patch_list = patchwork + .get_patch_query_num(&args.flag_project, args.flag_count as usize) + .unwrap_or_else(|err| panic!("Failed to obtain patch list: {}", err)); + } else { + patch_list = patchwork + .get_patch_query(&args.flag_project) + .unwrap_or_else(|err| panic!("Failed to obtain patch list: {}", err)); + } info!("snowpatch is ready to test new revisions from Patchwork."); for patch in patch_list { // If it's already been tested, we can skip it @@ -481,13 +488,9 @@ fn main() { patchwork.post_test_result(result, &patch.checks).unwrap(); } } - if args.flag_count > 0 { - patch_count += 1; - debug!("Tested {} patches out of {}", patch_count, args.flag_count); - if patch_count >= args.flag_count { - break 'daemon; - } - } + } + if args.flag_count > 0 { + break; } info!("Finished testing new revisions, sleeping."); thread::sleep(Duration::new(settings.patchwork.polling_interval * 60, 0)); diff --git a/src/patchwork.rs b/src/patchwork.rs index c9c335472496..31ac8ef95911 100644 --- a/src/patchwork.rs +++ b/src/patchwork.rs @@ -25,7 +25,9 @@ use std::result::Result; use tempdir::TempDir; use reqwest; -use reqwest::header::{qitem, Accept, Authorization, Basic, Connection, ContentType, Headers}; +use reqwest::header::{ + qitem, Accept, Authorization, Basic, Connection, ContentType, Headers, Link, RelationType, +}; use reqwest::Client; use reqwest::Response; use reqwest::StatusCode; @@ -306,6 +308,43 @@ impl PatchworkServer { .unwrap_or_else(|err| panic!("Failed to connect to Patchwork: {}", err))) } + fn get_next_link(&self, resp: &Response) -> Option { + let next = resp.headers().get::()?; + for val in next.values() { + if let Some(rel) = val.rel() { + if rel.iter().any(|reltype| reltype == &RelationType::Next) { + return Some(val.link().to_string()); + } + } + } + None + } + + pub fn get_patch_query_num( + &self, + project: &str, + num_patches: usize, + ) -> Result, serde_json::Error> { + let mut list: Vec = vec![]; + let mut url = Some(format!( + "{}{}/patches/{}&project={}", + &self.url, PATCHWORK_API, PATCHWORK_QUERY, project + )); + + while let Some(real_url) = url { + let resp = self.get_url(&real_url) + .unwrap_or_else(|err| panic!("Failed to connect to Patchwork: {}", err)); + url = self.get_next_link(&resp); + let new_patches: Vec = serde_json::from_reader(resp)?; + list.extend(new_patches); + if list.len() >= num_patches { + break; + } + } + list.truncate(num_patches); + Ok(list) + } + pub fn get_patch_dependencies(&self, patch: &Patch) -> Vec { // We assume the list of patches in a series are in order. let mut dependencies: Vec = vec![];