Working with Paginated Responses in Go

Working with Paginated Responses

Welcome back on your journey of mastering API interactions with Go! Building on your existing knowledge, today we will delve into API pagination . Pagination is a crucial concept for efficiently managing and retrieving large datasets through APIs. It allows you to request data in smaller, more manageable chunks instead of receiving all the data at once, which can be overwhelming and inefficient. Pagination is commonly used in scenarios involving extensive datasets, such as user lists or product catalogs, ensuring resources are used effectively and enhancing the application's performance.

Understanding Pagination Parameters

When you're dealing with a lot of data from an API, pagination is your friend. It helps you break down a massive dataset into smaller pieces that are easier to handle. To make this possible, there are two key settings you need to know about: page : This tells you which chunk of data you're looking at. Think of it as turning the page in a book to see the next set of information. limit : This controls how many items you see on each page, like deciding how many lines appear on each page of the book. By adjusting these parameters, you can control the flow of data retrieval. This lesson will build on what you've already learned about API requests, teaching you how to use these pagination settings effectively.

Fetching Paginated Data: Step-by-Step Example

Fetching paginated data involves several key steps. Below, we break down the process into smaller steps, explaining what each function does and how they work together to retrieve paginated data efficiently.

Step 1: Defining the Data Structure

Before fetching data, we need to define a struct that matches the JSON response from the API. This ensures the retrieved data can be correctly stored and used in Go. Go type Todo struct {ID int `json:"id"` Title string `json:"title"` Done bool `json:"done"`} ID stores the task’s unique identifier. Title represents the task’s description. Done tracks whether the task has been completed.

Step 2: Setting Up the Main Function

Join the 1M+ learners on CodeSignal

Be a part of our community of 1M+ users who develop and demonstrate their skills on CodeSignal

func main() {
    baseURL := "http://localhost:8000"
    fetchAllTodos(baseURL)
}

func main() {
    baseURL := "http://localhost:8000"
    fetchAllTodos(baseURL)
}

func fetchAllTodos(baseURL string) {
    page := 1 // Start from the first page

    for {
        todos, err := fetchTodosPage(baseURL, page)
        if err != nil {
            fmt.Printf("Error fetching page %d: %v", page, err)
            break
        }

        if len(todos) == 0 { // Exit loop if no more data
            break
        }

        fmt.Printf("Page %d fetched successfully!", page)
        printTodos(todos)

        page++ // Advance to next page
    }
}

func fetchAllTodos(baseURL string) {
    page := 1 // Start from the first page

    for {
        todos, err := fetchTodosPage(baseURL, page)
        if err != nil {
            fmt.Printf("Error fetching page %d: %v", page, err)
            break
        }

        if len(todos) == 0 { // Exit loop if no more data
            break
        }

        fmt.Printf("Page %d fetched successfully!", page)
        printTodos(todos)

        page++ // Advance to next page
    }
}

func fetchTodosPage(baseURL string, page int) ([]Todo, error) {
    // Construct the request URL with query parameters
    reqURL, err := url.Parse(fmt.Sprintf("%s/todos", baseURL))
    if err != nil {
        return nil, fmt.Errorf("error parsing URL: %w", err)
    }

    query := reqURL.Query()
    query.Set("page", fmt.Sprintf("%d", page))
    query.Set("limit", "3")
    reqURL.RawQuery = query.Encode()

    // Send the GET request
    resp, err := http.Get(reqURL.String())
    if err != nil {
        return nil, fmt.Errorf("error making request: %w", err)
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        return nil, fmt.Errorf("HTTP error: %s", resp.Status)
    }

    var pageTodos []Todo
    if err := json.NewDecoder(resp.Body).Decode(&pageTodos); err != nil {
        return nil, fmt.Errorf("error decoding JSON: %w", err)
    }

    return pageTodos, nil
}

func fetchTodosPage(baseURL string, page int) ([]Todo, error) {
    // Construct the request URL with query parameters
    reqURL, err := url.Parse(fmt.Sprintf("%s/todos", baseURL))
    if err != nil {
        return nil, fmt.Errorf("error parsing URL: %w", err)
    }

    query := reqURL.Query()
    query.Set("page", fmt.Sprintf("%d", page))
    query.Set("limit", "3")
    reqURL.RawQuery = query.Encode()

    // Send the GET request
    resp, err := http.Get(reqURL.String())
    if err != nil {
        return nil, fmt.Errorf("error making request: %w", err)
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        return nil, fmt.Errorf("HTTP error: %s", resp.Status)
    }

    var pageTodos []Todo
    if err := json.NewDecoder(resp.Body).Decode(&pageTodos); err != nil {
        return nil, fmt.Errorf("error decoding JSON: %w", err)
    }

    return pageTodos, nil
}

func printTodos(todos []Todo) {
    for _, todo := range todos {
        fmt.Printf("- ID: %d: %s (Done: %t)", todo.ID, todo.Title, todo.Done)
    }
}

func printTodos(todos []Todo) {
    for _, todo := range todos {
        fmt.Printf("- ID: %d: %s (Done: %t)", todo.ID, todo.Title, todo.Done)
    }
}

package main

import (
    "encoding/json"
    "fmt"
    "net/http"
    "net/url"
)

type Todo struct {
    ID    int    `json:"id"`
    Title string `json:"title"`
    Done  bool   `json:"done"`
}

func main() {
    baseURL := "http://localhost:8000"
    fetchAllTodos(baseURL)
}

func fetchAllTodos(baseURL string) {
    page := 1 // Start from the first page

    for {
        todos, err := fetchTodosPage(baseURL, page)
        if err != nil {
            fmt.Printf("Error fetching page %d: %v\n", page, err)
            break
        }

        if len(todos) == 0 { // Exit loop if no more data
            break
        }

        fmt.Printf("Page %d fetched successfully!\n", page)
        printTodos(todos)

        page++ // Advance to next page
    }
}

func fetchTodosPage(baseURL string, page int) ([]Todo, error) {
    // Construct the request URL with query parameters
    reqURL, err := url.Parse(fmt.Sprintf("%s/todos", baseURL))
    if err != nil {
        return nil, fmt.Errorf("error parsing URL: %w", err)
    }

    // Set the Query parameters
    query := reqURL.Query()
    query.Set("page", fmt.Sprintf("%d", page))
    query.Set("limit", "3")
    reqURL.RawQuery = query.Encode()

    // Send the GET request
    resp, err := http.Get(reqURL.String())
    if err != nil {
        return nil, fmt.Errorf("error making request: %w", err)
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        return nil, fmt.Errorf("HTTP error: %s", resp.Status)
    }

    var pageTodos []Todo
    if err := json.NewDecoder(resp.Body).Decode(&pageTodos); err != nil {
        return nil, fmt.Errorf("error decoding JSON: %w", err)
    }

    return pageTodos, nil
}

func printTodos(todos []Todo) {
    for _, todo := range todos {
        fmt.Printf("- ID: %d: %s (Done: %t)\n", todo.ID, todo.Title, todo.Done)
    }
}

package main

import (
    "encoding/json"
    "fmt"
    "net/http"
    "net/url"
)

type Todo struct {
    ID    int    `json:"id"`
    Title string `json:"title"`
    Done  bool   `json:"done"`
}

func main() {
    baseURL := "http://localhost:8000"
    fetchAllTodos(baseURL)
}

func fetchAllTodos(baseURL string) {
    page := 1 // Start from the first page

    for {
        todos, err := fetchTodosPage(baseURL, page)
        if err != nil {
            fmt.Printf("Error fetching page %d: %v\n", page, err)
            break
        }

        if len(todos) == 0 { // Exit loop if no more data
            break
        }

        fmt.Printf("Page %d fetched successfully!\n", page)
        printTodos(todos)

        page++ // Advance to next page
    }
}

func fetchTodosPage(baseURL string, page int) ([]Todo, error) {
    // Construct the request URL with query parameters
    reqURL, err := url.Parse(fmt.Sprintf("%s/todos", baseURL))
    if err != nil {
        return nil, fmt.Errorf("error parsing URL: %w", err)
    }

    // Set the Query parameters
    query := reqURL.Query()
    query.Set("page", fmt.Sprintf("%d", page))
    query.Set("limit", "3")
    reqURL.RawQuery = query.Encode()

    // Send the GET request
    resp, err := http.Get(reqURL.String())
    if err != nil {
        return nil, fmt.Errorf("error making request: %w", err)
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        return nil, fmt.Errorf("HTTP error: %s", resp.Status)
    }

    var pageTodos []Todo
    if err := json.NewDecoder(resp.Body).Decode(&pageTodos); err != nil {
        return nil, fmt.Errorf("error decoding JSON: %w", err)
    }

    return pageTodos, nil
}

func printTodos(todos []Todo) {
    for _, todo := range todos {
        fmt.Printf("- ID: %d: %s (Done: %t)\n", todo.ID, todo.Title, todo.Done)
    }
}

text

Page 1 fetched successfully!
- ID: 1: Buy groceries (Done: false)
- ID: 2: Call mom (Done: true)
- ID: 3: Finish project report (Done: false)
Page 2 fetched successfully!
- ID: 4: Workout (Done: true)
- ID: 5: Read a book (Done: false)
- ID: 6: Plan weekend trip (Done: false)
Page 3 fetched successfully!
...

Page 1 fetched successfully!
- ID: 1: Buy groceries (Done: false)
- ID: 2: Call mom (Done: true)
- ID: 3: Finish project report (Done: false)
Page 2 fetched successfully!
- ID: 4: Workout (Done: true)
- ID: 5: Read a book (Done: false)
- ID: 6: Plan weekend trip (Done: false)
Page 3 fetched successfully!
...

func main() {
    baseURL := "http://localhost:8000"
    fetchAllTodos(baseURL)
}

func fetchAllTodos(baseURL string) {
    page := 1 // Start from the first page

    for {
        todos, err := fetchTodosPage(baseURL, page)
        if err != nil {
            fmt.Printf("Error fetching page %d: %v", page, err)
            break
        }

        if len(todos) == 0 { // Exit loop if no more data
            break
        }

        fmt.Printf("Page %d fetched successfully!", page)
        printTodos(todos)

        page++ // Advance to next page
    }
}

func fetchTodosPage(baseURL string, page int) ([]Todo, error) {
    // Construct the request URL with query parameters
    reqURL, err := url.Parse(fmt.Sprintf("%s/todos", baseURL))
    if err != nil {
        return nil, fmt.Errorf("error parsing URL: %w", err)
    }

    query := reqURL.Query()
    query.Set("page", fmt.Sprintf("%d", page))
    query.Set("limit", "3")
    reqURL.RawQuery = query.Encode()

    // Send the GET request
    resp, err := http.Get(reqURL.String())
    if err != nil {
        return nil, fmt.Errorf("error making request: %w", err)
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        return nil, fmt.Errorf("HTTP error: %s", resp.Status)
    }

    var pageTodos []Todo
    if err := json.NewDecoder(resp.Body).Decode(&pageTodos); err != nil {
        return nil, fmt.Errorf("error decoding JSON: %w", err)
    }

    return pageTodos, nil
}

func printTodos(todos []Todo) {
    for _, todo := range todos {
        fmt.Printf("- ID: %d: %s (Done: %t)", todo.ID, todo.Title, todo.Done)
    }
}

package main

import (
    "encoding/json"
    "fmt"
    "net/http"
    "net/url"
)

type Todo struct {
    ID    int    `json:"id"`
    Title string `json:"title"`
    Done  bool   `json:"done"`
}

func main() {
    baseURL := "http://localhost:8000"
    fetchAllTodos(baseURL)
}

func fetchAllTodos(baseURL string) {
    page := 1 // Start from the first page

    for {
        todos, err := fetchTodosPage(baseURL, page)
        if err != nil {
            fmt.Printf("Error fetching page %d: %v\n", page, err)
            break
        }

        if len(todos) == 0 { // Exit loop if no more data
            break
        }

        fmt.Printf("Page %d fetched successfully!\n", page)
        printTodos(todos)

        page++ // Advance to next page
    }
}

func fetchTodosPage(baseURL string, page int) ([]Todo, error) {
    // Construct the request URL with query parameters
    reqURL, err := url.Parse(fmt.Sprintf("%s/todos", baseURL))
    if err != nil {
        return nil, fmt.Errorf("error parsing URL: %w", err)
    }

    // Set the Query parameters
    query := reqURL.Query()
    query.Set("page", fmt.Sprintf("%d", page))
    query.Set("limit", "3")
    reqURL.RawQuery = query.Encode()

    // Send the GET request
    resp, err := http.Get(reqURL.String())
    if err != nil {
        return nil, fmt.Errorf("error making request: %w", err)
    }
    defer resp.Body.Close()

    if resp.StatusCode != http.StatusOK {
        return nil, fmt.Errorf("HTTP error: %s", resp.Status)
    }

    var pageTodos []Todo
    if err := json.NewDecoder(resp.Body).Decode(&pageTodos); err != nil {
        return nil, fmt.Errorf("error decoding JSON: %w", err)
    }

    return pageTodos, nil
}

func printTodos(todos []Todo) {
    for _, todo := range todos {
        fmt.Printf("- ID: %d: %s (Done: %t)\n", todo.ID, todo.Title, todo.Done)
    }
}

text

Page 1 fetched successfully!
- ID: 1: Buy groceries (Done: false)
- ID: 2: Call mom (Done: true)
- ID: 3: Finish project report (Done: false)
Page 2 fetched successfully!
- ID: 4: Workout (Done: true)
- ID: 5: Read a book (Done: false)
- ID: 6: Plan weekend trip (Done: false)
Page 3 fetched successfully!
...

Working with Paginated Responses

Understanding Pagination Parameters

Fetching Paginated Data: Step-by-Step Example

Step 1: Defining the Data Structure

Step 2: Setting Up the Main Function

Join the 1M+ learners on CodeSignal

Step 3: Implementing Pagination Logic

Step 4: Fetching a Single Page

Step 5: Printing the Retrieved Data

Step 6: The Complete Flow

Paginated Output

Summary and Next Steps