Make Github API calls more robust + add issue matching tools

src/yatm_v2/Cargo.toml

@@ -9,6 +9,7 @@ edition = "2021"
 [dependencies]
 anyhow = "1.0.79"
 askama = "0.12.1"
+async-recursion = "1.0.5"
 chrono = "0.4.33"
 clap = { version = "4.4.18", features = ["derive"] }
 common = { path = "../common" }
@@ -19,3 +20,4 @@ percent-encoding = "2.3.1"
 serde = { version = "1.0.196", features = ["derive"] }
 serde_yaml = "0.9.31"
 tokio = { version = "1.36.0", features = ["full"] }
+url = "2.5.0"

src/yatm_v2/src/main.rs

@@ -101,26 +101,29 @@ fn demo_template() {
 async fn main() -> Result<()> {
     let gh = Github::new("paudrow".to_string(), "test-yatm-v2".to_string())?;
 
-    let issue_title = format!(
-        "My issue {}",
-        chrono::Local::now().format("%Y-%m-%d %H:%M:%S")
-    );
-    gh.create_issue(
-        issue_title,
-        "My issue body".to_string(),
-        vec![String::from("label")],
-    )
-    .await?;
-    // gh.close_all_issues().await?;
-
-    let issues = gh.get_issues().await?;
-    for issue in issues {
-        println!(
-            "{}, {}",
-            issue.title,
-            issue.body.unwrap_or("No body".into())
+    for _ in 0..300 {
+        let issue_title = format!(
+            "My issue {}",
+            chrono::Local::now().format("%Y-%m-%d %H:%M:%S")
         );
+        gh.create_issue(
+            issue_title.clone(),
+            "My issue body".to_string(),
+            vec![String::from("label")],
+        )
+        .await?;
     }
 
+    // let issues = gh.get_issues().await?;
+    // for issue in &issues {
+    //     println!(
+    //         "{}, {}",
+    //         issue.title,
+    //         issue.body.clone().unwrap_or("No body".into())
+    //     );
+    // }
+    // println!("Total issues: {}", &issues.len());
+    // gh.close_all_issues().await?;
+
     Ok(())
 }

src/yatm_v2/src/utils/github.rs

@@ -1,6 +1,11 @@
-use anyhow::{Context, Result};
+use anyhow::{anyhow, Context, Result};
+use async_recursion::async_recursion;
 use dotenv::dotenv;
+use octocrab::models::issues::Issue;
+use octocrab::models::IssueState;
+use octocrab::params::State;
 use octocrab::Octocrab;
+use tokio::time::{sleep, Duration};
 
 pub struct Github {
     octocrab: Octocrab,
@@ -60,54 +65,92 @@ impl Github {
         Ok(())
     }
 
-    pub async fn get_issues(&self) -> Result<octocrab::Page<octocrab::models::issues::Issue>> {
-        let issues = self
+    pub async fn get_issues(&self, state: Option<State>) -> Result<Vec<Issue>> {
+        let mut page: u32 = 1;
+        let mut issues: Vec<Issue> = Vec::new();
+        loop {
+            let page_issues = self.get_issues_helper(page, state).await?;
+            if page_issues.is_empty() {
+                break;
+            }
+            issues.extend(page_issues);
+            page += 1;
+        }
+        Ok(issues)
+    }
+
+    async fn get_issues_helper(&self, page: u32, state: Option<State>) -> Result<Vec<Issue>> {
+        let page = self
             .octocrab
             .issues(&self.owner, &self.repo)
             .list()
-            .state(octocrab::params::State::Open)
+            .per_page(100)
+            .state(state.unwrap_or(State::Open))
+            .page(page)
             .send()
             .await
             .context("Failed to list issues")?;
-        Ok(issues)
+        Ok(page.items)
     }
 
+    #[async_recursion]
     pub async fn create_issue(
         &self,
         title: String,
         body: String,
         labels: Vec<String>,
     ) -> Result<()> {
-        self.octocrab
+        let result = self
+            .octocrab
             .issues(&self.owner, &self.repo)
             .create(&title)
             .body(&body)
-            .labels(labels)
+            .labels(labels.clone())
             .send()
-            .await
-            .context(format!("Failed to create issue: {}", &title))?;
-        Ok(())
+            .await;
+
+        if result.is_ok() {
+            return Ok(());
+        }
+
+        let error = result.unwrap_err();
+        if let octocrab::Error::GitHub { source, .. } = &error {
+            if source
+                .message
+                .contains("You have exceeded a secondary rate limit")
+            {
+                println!("Secondary rate limit exceeded, waiting 60 seconds and retrying");
+                sleep(Duration::from_secs(60)).await;
+                // Retry the request by calling the function recursively.
+                return self.create_issue(title, body, labels).await;
+            }
+        }
+        Err(anyhow!(error))
     }
 
     pub async fn close_all_issues(&self) -> Result<()> {
         let issues = self
-            .octocrab
-            .issues(&self.owner, &self.repo)
-            .list()
-            .state(octocrab::params::State::Open)
-            .send()
+            .get_issues(Some(State::Open))
             .await
-            .context("Failed to list issues")?;
+            .context("Failed to get issues")?;
 
         for issue in issues {
-            self.octocrab
-                .issues(&self.owner, &self.repo)
-                .update(issue.number)
-                .state(octocrab::models::IssueState::Closed)
-                .send()
-                .await
-                .context(format!("Failed to delete issue: {}", issue.number))?;
+            self.close_issue(issue).await?;
         }
         Ok(())
     }
+
+    pub async fn close_issue(&self, issue: Issue) -> Result<()> {
+        self.octocrab
+            .issues(&self.owner, &self.repo)
+            .update(issue.number)
+            .state(IssueState::Closed)
+            .send()
+            .await
+            .context(format!(
+                "Failed to delete issue: #{} {}",
+                issue.number, issue.title
+            ))?;
+        Ok(())
+    }
 }