Skip to content

Commit 1ada2f1

Browse files
authored
Merge pull request #47 from syphar/github-fast-path
use github fast path to check for changes before doing the git pull
2 parents 674bdfc + 3bd1c11 commit 1ada2f1

File tree

3 files changed

+150
-27
lines changed

3 files changed

+150
-27
lines changed

Cargo.toml

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,6 @@ edition = "2018"
1111
readme = "changelog.md"
1212
include = ["src/**/*", "LICENSE.md", "README.md", "CHANGELOG.md"]
1313

14-
[lib]
15-
test = false
16-
1714
[[test]]
1815
name = "baseline"
1916
path = "tests/baseline.rs"
@@ -46,6 +43,7 @@ bstr = "1.0.1"
4643
thiserror = "1.0.32"
4744
ahash = "0.8.0"
4845
hashbrown = { version = "0.14.0", features = ["raw"] }
46+
reqwest = { version = "0.12", features = ["blocking"] }
4947

5048
[dev-dependencies]
5149
gix-testtools = "0.15.0"

src/index/diff/github.rs

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
use reqwest::{StatusCode, Url};
2+
3+
#[derive(Debug)]
4+
pub(crate) enum FastPath {
5+
UpToDate,
6+
NeedsFetch,
7+
Indeterminate,
8+
}
9+
10+
/// extract username & repository from a fetch URL, only if it's on Github.
11+
fn user_and_repo_from_url_if_github(fetch_url: &gix::Url) -> Option<(String, String)> {
12+
let url = Url::parse(&fetch_url.to_string()).ok()?;
13+
if !(url.host_str() == Some("github.com")) {
14+
return None;
15+
}
16+
17+
// This expects GitHub urls in the form `github.com/user/repo` and nothing
18+
// else
19+
let mut pieces = url.path_segments()?;
20+
let username = pieces.next()?;
21+
let repository = pieces.next()?;
22+
let repository = repository.strip_suffix(".git").unwrap_or(repository);
23+
if pieces.next().is_some() {
24+
return None;
25+
}
26+
Some((username.to_string(), repository.to_string()))
27+
}
28+
29+
/// use github fast-path to check if the repository has any changes
30+
/// since the last seen reference.
31+
///
32+
/// To save server side resources on github side, we can use an API
33+
/// to check if there are any changes in the repository before we
34+
/// actually run `git fetch`.
35+
///
36+
/// On non-github fetch URLs we don't do anything and always run the fetch.
37+
///
38+
/// Code gotten and adapted from
39+
/// https://github.com/rust-lang/cargo/blob/edd36eba5e0d6e0cfcb84bd0cc651ba8bf5e7f83/src/cargo/sources/git/utils.rs#L1396
40+
///
41+
/// GitHub documentation:
42+
/// https://docs.github.com/en/rest/commits/commits?apiVersion=2022-11-28#get-a-commit
43+
/// specifically using `application/vnd.github.sha`
44+
pub(crate) fn has_changes(
45+
fetch_url: &gix::Url,
46+
last_seen_reference: &gix::ObjectId,
47+
branch_name: &str,
48+
) -> Result<FastPath, reqwest::Error> {
49+
let (username, repository) = match user_and_repo_from_url_if_github(fetch_url) {
50+
Some(url) => url,
51+
None => return Ok(FastPath::Indeterminate),
52+
};
53+
54+
let url = format!(
55+
"https://api.github.com/repos/{}/{}/commits/{}",
56+
username, repository, branch_name,
57+
);
58+
59+
let client = reqwest::blocking::Client::builder()
60+
.user_agent("crates-index-diff")
61+
.build()?;
62+
let response = client
63+
.get(&url)
64+
.header("Accept", "application/vnd.github.sha")
65+
.header("If-None-Match", format!("\"{}\"", last_seen_reference))
66+
.send()?;
67+
68+
let status = response.status();
69+
if status == StatusCode::NOT_MODIFIED {
70+
Ok(FastPath::UpToDate)
71+
} else if status.is_success() {
72+
Ok(FastPath::NeedsFetch)
73+
} else {
74+
// Usually response_code == 404 if the repository does not exist, and
75+
// response_code == 422 if exists but GitHub is unable to resolve the
76+
// requested rev.
77+
Ok(FastPath::Indeterminate)
78+
}
79+
}
80+
81+
#[cfg(test)]
82+
mod tests {
83+
use super::*;
84+
use std::convert::TryFrom;
85+
86+
#[test]
87+
fn test_github_http_url() {
88+
let (user, repo) = user_and_repo_from_url_if_github(
89+
&gix::Url::try_from("https://github.com/some_user/some_repo.git").unwrap(),
90+
)
91+
.unwrap();
92+
assert_eq!(user, "some_user");
93+
assert_eq!(repo, "some_repo");
94+
}
95+
96+
#[test]
97+
fn test_github_ssh_url() {
98+
let (user, repo) = user_and_repo_from_url_if_github(
99+
&gix::Url::try_from("ssh://[email protected]/some_user/some_repo.git").unwrap(),
100+
)
101+
.unwrap();
102+
assert_eq!(user, "some_user");
103+
assert_eq!(repo, "some_repo");
104+
}
105+
106+
#[test]
107+
fn test_non_github_url() {
108+
assert!(user_and_repo_from_url_if_github(
109+
&gix::Url::try_from("https://not_github.com/some_user/some_repo.git").unwrap(),
110+
)
111+
.is_none());
112+
}
113+
}

src/index/diff/mod.rs

Lines changed: 36 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ use gix::prelude::ObjectIdExt;
44
use std::sync::atomic::AtomicBool;
55

66
mod delegate;
7+
mod github;
78

89
use delegate::Delegate;
910

@@ -66,6 +67,8 @@ pub enum Error {
6667
name: String,
6768
mappings: Vec<gix::remote::fetch::Mapping>,
6869
},
70+
#[error("Error when fetching GitHub fastpath.")]
71+
GithubFetch(#[from] reqwest::Error),
6972
}
7073

7174
/// Find changes without modifying the underling repository
@@ -175,30 +178,39 @@ impl Index {
175178
.replace_refspecs(Some(spec.as_str()), gix::remote::Direction::Fetch)
176179
.expect("valid statically known refspec");
177180
}
178-
let res: gix::remote::fetch::Outcome = remote
179-
.connect(gix::remote::Direction::Fetch)?
180-
.prepare_fetch(&mut progress, Default::default())?
181-
.receive(&mut progress, should_interrupt)?;
182-
let branch_name = format!("refs/heads/{}", self.branch_name);
183-
let local_tracking = res
184-
.ref_map
185-
.mappings
186-
.iter()
187-
.find_map(|m| match &m.remote {
188-
gix::remote::fetch::Source::Ref(r) => (r.unpack().0 == branch_name)
189-
.then_some(m.local.as_ref())
190-
.flatten(),
191-
_ => None,
192-
})
193-
.ok_or_else(|| Error::NoMatchingBranch {
194-
name: branch_name,
195-
mappings: res.ref_map.mappings.clone(),
196-
})?;
197-
self.repo
198-
.find_reference(local_tracking)
199-
.expect("local tracking branch exists if we see it here")
200-
.id()
201-
.detach()
181+
182+
let (url, _) = remote.sanitized_url_and_version(gix::remote::Direction::Fetch)?;
183+
if matches!(
184+
github::has_changes(&url, &from, self.branch_name)?,
185+
github::FastPath::UpToDate
186+
) {
187+
from.clone()
188+
} else {
189+
let res: gix::remote::fetch::Outcome = remote
190+
.connect(gix::remote::Direction::Fetch)?
191+
.prepare_fetch(&mut progress, Default::default())?
192+
.receive(&mut progress, should_interrupt)?;
193+
let branch_name = format!("refs/heads/{}", self.branch_name);
194+
let local_tracking = res
195+
.ref_map
196+
.mappings
197+
.iter()
198+
.find_map(|m| match &m.remote {
199+
gix::remote::fetch::Source::Ref(r) => (r.unpack().0 == branch_name)
200+
.then_some(m.local.as_ref())
201+
.flatten(),
202+
_ => None,
203+
})
204+
.ok_or_else(|| Error::NoMatchingBranch {
205+
name: branch_name,
206+
mappings: res.ref_map.mappings.clone(),
207+
})?;
208+
self.repo
209+
.find_reference(local_tracking)
210+
.expect("local tracking branch exists if we see it here")
211+
.id()
212+
.detach()
213+
}
202214
};
203215

204216
Ok((

0 commit comments

Comments
 (0)