import dotenv from "dotenv";
dotenv.config();

import axios from "axios";

const CD_AUTOMATION_API_KEY = process.env.CD_AUTOMATION_API_KEY;
const CD_AUTOMATION_WORKSPACE_ID = process.env.CD_AUTOMATION_WORKSPACE_ID;
const AUTOMATION_API_ROUTE = "https://automationv2.captaindata.co";

/**
 * Function to fetch output fields.
 * @param {*} appSlug
 * @param {*} actionSlug
 * @returns An array of keys.
 */
async function fetchOutputFields(appSlug, actionSlug) {
  const response = await axios.get(
    `${AUTOMATION_API_ROUTE}/v1/outputs?template=${appSlug}/${actionSlug}`,
    {
      headers: {
        Authorization: `x-api-key ${CD_AUTOMATION_API_KEY}`,
        "x-project-id": CD_AUTOMATION_WORKSPACE_ID,
      },
    }
  );

  const mappedArray = response.data.map((item) => item.name);
  return mappedArray;
}

/**
 * Function to fetch input fields schema.
 * @param {*} appSlug
 * @param {*} actionSlug
 * @returns An array of keys.
 */
async function fetchInputSchema(appSlug, actionSlug) {
  const response = await axios.get(
    `${AUTOMATION_API_ROUTE}/v1/inputs/rules?template=${appSlug}/${actionSlug}`,
    {
      headers: {
        Authorization: `x-api-key ${CD_AUTOMATION_API_KEY}`,
        "x-project-id": CD_AUTOMATION_WORKSPACE_ID,
      },
    }
  );

  const rawMappingData = response.data;

  // Function to map the data to a simpler structure
  const mappedData = Object.keys(rawMappingData.mapping).map((key) => {
    const item = rawMappingData.mapping[key];
    return {
      key,
      regex: item.regex || {},
      help: item.help || "",
      placeholder: item.placeholder || "",
      required: item.required === 1, // Convert 1 to true, 0 to false
      primary: item.primary || false, // Mark as primary if it exists, else false
    };
  });

  return mappedData;
}

export { fetchOutputFields, fetchInputSchema };


import { google } from "googleapis";

const SHEET_ID = "1AOjKaIegrph7cCeFGZn0_lzjBR7bWZtSOMkxvfJx7Vc";

// Define the headers for mapping
const headers = [
  "id",
  "added",
  "ai_content",
  "mint_permalink",
  "permalink",
  "mdx_content",
  "name",
  "app",
  "description",
  "type",
  "created_at",
  "uid",
  "action_verb",
  "action_unit",
  "inputs_readme",
  "parameters_readme",
  "input_parameters_schema",
  "overview",
  "is_rotatable",
];

// Function to fetch data from Google Sheets
async function fetchGoogleSheetData() {
  // Service account in base64 because of issue with deployment
  const decodedServiceAccount = Buffer.from(
    process.env.GOOGLE_SHEETS_CREDENTIALS,
    "base64"
  ).toString("utf8");
  const serviceAccount = JSON.parse(decodedServiceAccount);

  const auth = new google.auth.GoogleAuth({
    credentials: serviceAccount,
    scopes: ["https://www.googleapis.com/auth/spreadsheets"],
  });

  const sheets = google.sheets({ version: "v4", auth });

  const range = "actions!A:S"; // Specify the range to cover columns A to R
  const response = await sheets.spreadsheets.values.get({
    spreadsheetId: SHEET_ID,
    range: range,
  });

  return response.data.values;
}

// Function to pad or truncate rows to match the header length
function ensureRowLength(row) {
  // If row length is less than headers, pad with empty strings (or null)
  if (row.length < headers.length) {
    const missingValues = headers.length - row.length;
    row = [...row, ...new Array(missingValues).fill("")]; // Pad missing values with empty strings
  }

  // If row length is greater than headers, truncate to match headers length
  if (row.length > headers.length) {
    console.warn(`Row has more values than expected. Truncating row: ${row}`);
    row = row.slice(0, headers.length); // Truncate the row
  }

  return row;
}

// Function to map each row to headers and ensure correct length
function mapDataToHeaders(row) {
  row = ensureRowLength(row);

  // Now that the row length matches, map it to headers
  const mappedData = headers.reduce((acc, header, index) => {
    acc[header] = row[index];
    return acc;
  }, {});

  return mappedData;
}

// Filter rows based on 'type' and 'ai_content'
async function filterSheetData() {
  const data = await fetchGoogleSheetData();

  // Remove header row (assuming the first row contains headers)
  const rows = data.slice(1);

  // Map the rows to header names and ensure they all have the same length
  const mappedRows = rows.map((row) => mapDataToHeaders(row));

  // Filter rows based on 'type' and 'ai_content'
  const filteredRows = mappedRows.filter(
    (row) =>
      row &&
      row.type === "javascript" &&
      row.ai_content === "FALSE" &&
      row.mint_permalink !== ""
  );

  return filteredRows;
}

export { filterSheetData };


// Load environment variables
import dotenv from "dotenv";
dotenv.config();

import fs from "fs";
import path from "path";

import { fetchOutputFields, fetchInputSchema } from "./captain.service.js";
import { filterSheetData } from "./gsheets.service.js";
import { sendToOpenAI } from "./openai.service.js";
import { fetchZendeskContent } from "./zendesk.service.js";

// Function to write the documentation update locally without committing
async function updateDocumentation(formattedContent, filePath, name) {
  try {
    // Ensure the path is correct by resolving the full path
    const fileFullPath = path.resolve(`../${filePath}.mdx`);

    // Check if the file exists before writing
    if (fs.existsSync(fileFullPath)) {
      fs.writeFileSync(fileFullPath, formattedContent, "utf8");
      console.log(
        `Successfully updated documentation ${name} file at ${fileFullPath}`
      );
    } else {
      console.error(`File at ${fileFullPath} does not exist`);
    }
  } catch (error) {
    console.error("Error writing documentation:", error);
  }
}

// Main function to tie everything together
export default async function main() {
  try {
    // Step 1: Filter the data from Google Sheets
    const filteredData = await filterSheetData();

    console.log(`Working on ${filteredData.length} actions.`);
    for (const row of filteredData) {
      const {
        name,
        description,
        inputs_readme,
        parameters_readme,
        input_parameters_schema,
        mint_permalink,
        permalink,
        app,
      } = row;

      let zendeskContent = [];

      // Step 2: Fetch Zendesk Content
      // Check if either inputs_readme or parameters_readme has elements
      if (inputs_readme.length > 0 || parameters_readme.length > 0) {
        console.log(`Fetching documentation for ${name}`);

        // Safely parse inputs_readme and parameters_readme, ensuring they are valid JSON strings
        let parsedInputs = [];
        let parsedParameters = [];

        try {
          if (inputs_readme.length > 0) {
            parsedInputs = JSON.parse(inputs_readme);
          }
        } catch (e) {
          console.error("Error parsing inputs_readme:", e);
        }

        try {
          if (parameters_readme.length > 0) {
            parsedParameters = JSON.parse(parameters_readme);
          }
        } catch (e) {
          console.error("Error parsing parameters_readme:", e);
        }

        // Combine the two parsed arrays
        const combinedReadme = [...parsedInputs, ...parsedParameters];

        if (combinedReadme.length > 0) {
          // Only call fetchZendeskContent if there are valid items to fetch
          zendeskContent = await fetchZendeskContent(combinedReadme);
        } else {
          console.log("No valid articles to fetch from Zendesk.");
        }
      }

      // Step 3: Fetch API input/output schemas and parameters
      const outputFields = await fetchOutputFields(app, permalink);
      const inputFields = await fetchInputSchema(app, permalink);
      //   const parameters = await fetchParameters(name);

      // Step 4: Combine all data
      const data = {
        name,
        description,
        parameters: input_parameters_schema,
        inputFields,
        outputFields,
        articles: zendeskContent,
      };

      // Step 5: Send to OpenAI for formatting
      const formattedContent = await sendToOpenAI(data);
      //   console.log("formattedContent", formattedContent);

      // Step 6: Update the documentation in Mintlify
      await updateDocumentation(formattedContent, mint_permalink, name);

      // One at a time for debugging & QA
      //   break;
    }
  } catch (error) {
    console.error("Error processing data:", error);
  }
}

main();


import openai from "openai";

const OPENAI_API_KEY = process.env.OPENAI_API_KEY;

// Function to send data to OpenAI for formatting
async function sendToOpenAI(data) {
  const openaiClient = new openai.OpenAI({ apiKey: OPENAI_API_KEY });

  const prompt = `
    Format the following documentation for a technical API. Make it clean, concise, and organized.
    - Name: ${data.name}
    - Description: ${data.description}
    - Input Fields: ${JSON.stringify(data.inputFields)}
    - Output Fields: ${JSON.stringify(data.outputFields)}
    - Parameters: ${JSON.stringify(data.parameters)}
    - Existing articles: ${JSON.stringify(data.articles)}

    You are a content generator for API documentation. Your task is to generate the content that follows the format below, starting with:

    ---
    title: "The name of the action"
    description: "A brief description of what the action does"
    ---

    Focus strictly on structure and documentation.
    **Do not wrap the content inside \`\`\`mdx \`\`\`**.

    Always structure content as:
    - Introduction / use cases / explanatory videos (don't add a title 'Introduction' just add directly the content)
    - Inputs (use Input fields)
    - Parameters (user Parameters)
    - Output fields (use Output fields)
    - Specificities (include only if there's new value-added content)

    Replace:
    - 'task(s)' in the context of automating with 'action'
    - 'task(s)' in the context of consumption/usage with 'credit(s)'

    There's no need to specify phrases like 'To use this setting with the API' since it's implied that it will be used via API.

    Avoid repetition. For example, if you list the inputs in bullet points, do not repeat the input format in JSON.

    Use these **Mintlify components** (don't overuse them):

    1. For 'Input Fields' and 'Parameters' (e.g., 'max_results', 'excludeViewedLeads') use **ParamField**: 
        <ParamField path='param' type='string'>
            An example parameter field
        </ParamField>

        Properties:
        - 'type': Expected type of the parameter's value
        - 'required': Boolean indicating if the parameter is required
        - 'deprecated': Boolean indicating if the parameter is deprecated
        - 'default': Default value if no value is provided
        - 'initialValue': Initial value for the playground
        - 'placeholder': Placeholder text for the input
        - 'children': Description of the parameter (markdown enabled)
        **Do not** add an extra title like '### Maximum number of results'; the ParamField component itself is enough.

    2. For 'Output Fields', use the 'ResponseField' component inside the 'Expandable' component. 
        Key Rule: Always use one main ResponseField with one Expandable component. Inside the Expandable, add individual ResponseField components for each field. **DO NOT** list each ResponseField individually.
        Sort the fields in ascending order by name.
        For example:
        <ResponseField>
            <Expandable title='properties'>
                <ResponseField name='linkedin_company_url' type='string'>The LinkedIn company URL</ResponseField>
                <ResponseField name='industry' type='string'>The industry of the company</ResponseField>
                <ResponseField name='followers_count' type='integer'>The number of followers on LinkedIn</ResponseField>

                <ResponseField name='affiliate_status' type='boolean'>The affiliate status of the company</ResponseField>
            </Expandable>
        </ResponseField>

        Also add an 'Accordion' with the JSON - Map the array to a JSON object with keys.
        Example:
        <Accordion title="Show JSON Output Schema">
            \`\`\`json
                {{Output Fields}}
            \`\`\`
        </Accordion>
        Separate the two section by a brief sentence introducting the JSON schema.

    3. To describe the automation setup you can use <Steps></Steps> for step-by-step processes description:
        Example:
        <Steps>
            <Step title='First Step'>Instructions for the first step.</Step>
            <Step title='Second Step'>Instructions for the second step.</Step>
        </Steps>

    4. **Tooltip**: Use <Tooltip tip='This is a tooltip!'>Hover over me</Tooltip> for additional content like technical definitions.

    5. **Highlight content** whenever possible if something is important and worth highlighting:
        - <Note>This adds a note in the content</Note>
        - <Warning>This raises a warning</Warning>
        - <Info>This highlights important information</Info>
        - <Tip>This suggests a helpful tip</Tip>
        - <Check>This indicates a successful or completed action</Check>

    6. **Embedding Iframes**: For embedded content like YouTube or Loom videos:
        <iframe src='{{url}}' frameborder='0' width='100%' height='400px' allow='accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture' allowfullscreen></iframe>

        **Always**: When you see iframe links containing 'https://app.guideflow.com/embed/', extract the link and use it as:
        <iframe frameborder='0' width='100%' height='400px' allowfullscreen src='{{url}}'></iframe>
        **ONLY** do that if you detect links containing 'https://app.guideflow.com/embed/'.

        Example:
        Replace: <iframe style='overflow: hidden; position: absolute; border: none;' src='https://app.guideflow.com/embed/{guide-id}' width='100%' height='100%' scrolling='no' allowfullscreen=''></iframe>
        With: <iframe frameborder='0' width='100%' height='460px' allowfullscreen src='https://app.guideflow.com/embed/{guide-id}'></iframe>

        Add a brief description to introduce each video or iframe. **Do not** use 'Videos' as a title—incorporate the video within the context.

    7. **Code component**: For displaying code snippets in various languages (e.g., JSON, JavaScript).
        Example:
        \`\`\`json
        {
            'key': 'abc',
            'other_key': 123
        }
        \`\`\`

    The generated content should be clear, technical, and organized, without unnecessary context or explanations—just the content.
    `;

  // Change to use the chat-based completions API (v1/chat/completions)
  const response = await openaiClient.chat.completions.create({
    model: "gpt-4o", // Use the latest GPT-4 model
    messages: [
      {
        role: "system",
        content: "You are an assistant that formats API documentation.",
      },
      {
        role: "user",
        content: prompt,
      },
    ],
    temperature: 0, // Adjust temperature based on your preference
  });

  return response.choices[0].message.content.trim();
}

export { sendToOpenAI };


import dotenv from "dotenv";
import axios from "axios";

// Load environment variables
dotenv.config();

// Ensure the Zendesk API Key is set in the environment variables
const ZENDESK_API_KEY = process.env.ZENDESK_API_KEY;

// Function to fetch Zendesk data
async function fetchZendeskContent(articleIds) {
  // Make sure articleIds is now an array and proceed with mapping
  if (Array.isArray(articleIds)) {
    const contentPromises = articleIds.map(async (id) => {
      // Use Node.js Buffer to encode the API key (equivalent of btoa in browsers)
      const authorizationData =
        "Basic " + Buffer.from(ZENDESK_API_KEY).toString("base64");

      // Make sure the URL is correctly formatted
      const zendeskUrl = `https://support.captaindata.co/api/v2/help_center/articles/${id}`;

      // Fetch the Zendesk content using axios
      try {
        const response = await axios.get(zendeskUrl, {
          headers: {
            "Content-Type": "application/json",
            Authorization: authorizationData,
          },
        });

        return response.data.article.body;
      } catch (error) {
        console.error("Error fetching Zendesk article:", error.response.data);
        return null;
      }
    });

    // Return all the responses once all promises are resolved
    return Promise.all(contentPromises);
  } else {
    console.error("The articleIds input is not an array.");
    return [];
  }
}

export { fetchZendeskContent };


Inputs

Parameters

Output Fields

Specificities

Extracts data from a website, such as emails and social network URLs (LinkedIn, Twitter, etc.).

Website Scraper

Captain Data

Guides

API Reference

Actions

Platform

Changelog

Login

Support

Welcome to Captain Data's API implementation guide.

Quickstart

Understand key concepts and implement our APIs using coding best practices.

Core Concepts

Find answers to common questions and troubleshooting tips for using the Captain Data API, including error handling, best practices, and debugging guidance.

FAQ & Troubleshooting

Comprehensive guide to syncing users and accounts with the Captain Data API.

Adding & Managing Users

With Captain Data, you can sync your own LinkedIn user accounts or provision rented accounts.

When to Sync Users vs. Use Rented LinkedIn Accounts

Example section for showcasing API endpoints

Users & Accounts FAQ & Troubleshooting

A quick guide to automating workflows via Captain Data's API, covering workflow logic, parameters, and how to retrieve results efficiently.

Workflow Launch Logic & Tips for Developers

This guide outlines both methods, when to use them, and examples to help you determine the most suitable approach for processing inputs.

Inputs Processing: Single vs. Batch

This guide explains the different Run (Job) statuses in Captain Data and how to manage them.

Manage Runs & their Status

Comprehensive guide to handling API errors and retry mechanisms

Error & Retries Mechanism

Guide to managing and deleting scheduled runs using API endpoints

Manage Schedules

Create & Manage Webhooks

Polling vs. Waiting for Webhooks: Handling GET RESULTS

Polling Data

The importance of using LinkedIn Profile IDs over mutable handles for reliable data access and integration.

LinkedIn API Best Practices

Learn to manage LinkedIn cookies, user accounts, and licenses in Captain Data.

LinkedIn Accounts & Licences

Best practices to prevent LinkedIn restrictions with Captain Data.

LinkedIn Rate Limits

Best practices for building Chrome extensions tailored for LinkedIn and accessing source code examples.

Building Chrome Extensions for LinkedIn & Source Code

Comprehensive guide to implementing LinkedIn campaigns using automation.

Implementation Guide to Build LinkedIn Campaigns

Set up and automate workflows with Captain Data in Make, including tips for integrating CRMs, spreadsheets, and databases.

Launch a Workflow with Make

Step-by-step guide to fetching Captain Data job results using Make.

Get Run Results with Make

Set up and automate workflows with Captain Data in n8n.

Launch a Workflow with n8n

Step-by-step guide to fetching Captain Data job results using n8n.

Get Run Results with n8n

Step-by-step guide to scheduling Captain Data workflows using Zapier for seamless automation.

Launch a Workflow with Zapier

Step-by-step guide to fetching Captain Data job results using Zapier.

Get Run Results with Zapier

Introduction

Get Workspace

Create or Update a User

List all Users

Remove a User

Add or Update an Integration Account

Get an Integration's Account

List all Integration's Accounts

Remove an Account

Launch a Workflow

Get a Workflow

List all Workflows

Get a Run

Get a Run's Results

List all Workflow's Runs

Cancel a Run

Retry a Run

Remove a Schedule

Get a Run's Inputs

Retry a Webhook

Learn how to get started with automated actions to streamline your workflows.

This action allows you to extract a list of people from LinkedIn based on a set of given criteria.

Search LinkedIn People

Search LinkedIn Companies

Extract a list of employees from a specific LinkedIn company profile.

Search LinkedIn Company Employees

Extract posts data from a LinkedIn search content.

Search LinkedIn Content

Search LinkedIn Jobs

Search LinkedIn Groups

Search LinkedIn Events

Search LinkedIn Schools

Extracts information from a specific LinkedIn profile.

Extract LinkedIn People Profile

Extract information from a LinkedIn company page.

Extract LinkedIn Company Profile

Extracts a connection's contact information, such as phone numbers and email addresses.

Extract LinkedIn User Contact Info

This automation extracts the content and associated information from specific LinkedIn posts you input.

Extract LinkedIn Post Content

Extract LinkedIn Job Profile

Retrieve detailed information from LinkedIn Events, including descriptions, attendee numbers, and speaker details.

Extract LinkedIn Event

Extract a list of event attendees while being an admin or an attendee of the event.

Extract LinkedIn Event Attendees

Extract up to 2500 LinkedIn group's members.

Extract LinkedIn Group Members

Extract the list of your LinkedIn profile's followers.

Extract LinkedIn Followers

Extract all the followers from a company page (you need to be admin).

Extract LinkedIn Page Followers

Extract information from LinkedIn polls such as the total number of votes, the post content, and more.

Extract LinkedIn Polls Information

Extract detailed information about participants who responded to your LinkedIn poll. You must be the poll owner to access this data.

Extract LinkedIn Polls Participants

Extract all the affiliate company pages present on a LinkedIn company page.

Extract LinkedIn Company Affiliate Pages

This action allows you to extract companies that are similar to the companies you input.

Extract LinkedIn Similar Companies (lookalikes)

Extract a school's alumni given specific criteria.

Extract LinkedIn School Alumnis

Extract the profiles of people who liked a specific LinkedIn post.

Extract LinkedIn Post Likers

Extract LinkedIn profiles, comments, and names of everyone who commented on a LinkedIn post.

Extract LinkedIn Post Commenters

Extract the list of profiles that visited your LinkedIn profile.

Extract LinkedIn Profile Viewers

This action extracts all comment activities from a LinkedIn profile.

Extract LinkedIn People Comment Activity

This action extracts all the reactions (like, support, love, etc.) from a LinkedIn profile's activity.

Extract LinkedIn People Reaction Activity

Extract all the LinkedIn activity posts from a specific people profile.

Extract LinkedIn People Post Activity

Send a message to a LinkedIn profile (you must be connected).

Message LinkedIn Profile

Extracts conversations from your LinkedIn account.

Extract LinkedIn Conversations

Extract messages from a LinkedIn conversation.

Extract LinkedIn Messages

Connect with a LinkedIn user by sending a personalized message (optional).

Connect LinkedIn Profile

Extract LinkedIn Connections

Visits and extracts data from a specific LinkedIn profile.

Visit LinkedIn Profile

Automatically follow or unfollow a list of LinkedIn people profiles.

Follow LinkedIn People Profile

Engage with LinkedIn publications, articles, and posts.

Like LinkedIn Post

Invite LinkedIn contacts to your LinkedIn event.

Invite People to a LinkedIn Event

Automatically accept received invitations from your LinkedIn network.

Accept LinkedIn Received Invitations

Withdraws LinkedIn sent pending invitations.

Withdraw LinkedIn Pending Invitations

Search and extract profiles from a LinkedIn Recruiter Lite Search.

Search LinkedIn Recruiter Lite

This action extracts a list of leads based on a set of given criteria.

Search Sales Navigator Leads

Extract a list of companies ('Accounts') from a set of given criteria.

Search Sales Navigator Companies

Extract employees from a specific company on Sales Navigator.

Search Sales Navigator Company Employees

Extract leads from a Sales Navigator Leads List.

Extract Sales Navigator Leads List

Extract accounts (companies) from a list on Sales Navigator.

Extract Sales Navigator Accounts List

Retrieve a search URL for employee metrics starting from a company ID.

Search Sales Navigator Metrics

This action extracts the distribution of a company's employees by function from Sales Navigator, along with the growth of these functions over 3 months, 6 months, and 1 year.

Extract Sales Navigator Company Employee Distribution & Headcount

This action provides the evolution of a company's employee count over 6 months, 1 year, and 2 years, along with the total number of employees.

Extract Sales Navigator Company Employee Count

Sends a connection request when visiting a Sales Navigator People profile.

Connect Sales Navigator Profile

Visit and extract information from a specific LinkedIn profile on Sales Navigator.

Visit Sales Navigator People Profile

Visit and extract information from a LinkedIn Sales Navigator company page.

Visit Sales Navigator Company Profile

This action converts a Sales Navigator URL into a standard LinkedIn URL for either a user or a company.

Getting Started

LinkedIn

Sales Navigator

By Apps

Website Scraper

Inputs

Parameters

Output Fields

Specificities

Getting Started

LinkedIn

Sales Navigator

By Apps

​Inputs

​Parameters

​Output Fields

​Specificities

Inputs

Parameters

Output Fields

Specificities