# Appannie Crawler

function sleep(ms) {
  return new Promise((res) => setTimeout(res, ms));
}
function getDateString() {
  const now = new Date();
  now.setDate(now.getDate() - 1);
  return `${now.getFullYear()}-${now.getMonth() + 1}-${now.getDate()}`;
}
function extractCsrfToken() {
  return document.cookie
    .split('; ')
    .find((cookie) => cookie.startsWith('csrftoken'))
    .split('=')[1];
}
async function getTopCharts({
  market,
  countryCode,
  category,
  date = getDateString(),
}) {
  if (!countryCode || !market || !category) {
    return console.error(
      'Parameter `countryCode`, `market`, `category` are required!'
    );
  }
  const csrfToken = extractCsrfToken();
  const json = await fetch(
    `https://www.appannie.com/ajax/top-chart/table/?market=${market}&country_code=${countryCode}&category=${category}&date=${date}&rank_sorting_type=rank&page_size=100&order_type=desc`,
    {
      headers: {
        'x-csrftoken': csrfToken,
        'x-requested-with': 'XMLHttpRequest',
      },
    }
  ).then((res) => res.json());
  const {
    available,
    table: { rows },
  } = json;
  if (!available) {
    console.warn(
      `rows of ${market} ${countryCode} ${category} is not available.`
    );
    return [];
  }
  return rows;
}
function getDataAndSend({ country, date, os: oses, category: categories }) {
  const queue = oses.flatMap((os) => {
    return categories[os].map((category) => ({
      os,
      category,
    }));
  });
  let running = true;
  async function start() {
    for (const params of queue) {
      if (!running) break;
      const { os, category } = params;
      try {
        console.log(`Start fetching ${country} ${os} ${category}...`)
        const rows = await getTopCharts({
          countryCode: country,
          market: os,
          category,
          date,
        });
        if (rows.length > 0) {
          sendData({
            os,
            category_id: category,
            country,
            rows,
          });
        }
        await sleep(15000);
      } catch (e) {
        console.error(e);
        break;
      }
    }
  }
  start();
  return () => console.warn('Remaining tasks are stopped.');
}
function sendData(body) {
  return fetch('https://aixpert.appier.com/api/appannie/update_top_apps', {
    method: 'post',
    body: JSON.stringify(body),
  });
}

stop = getDataAndSend({
  category: {
    'google-play': [400000,400001,400002,400003,400004,400005,400006,400007,400008,400009,400010,400011,400012,400013,400014,400015,400016,400017,400018,400019,400020,400021,400022,400023,400024,400025,400026,400027,400028,400029,400030,400031,400032,400033,400034,400035,400036,400037,400038,400039,400040,400041,400042,400043,400044,400045,400046,400047,400048,400049,400050,400051,400052,400053,400054,400055,400056,400057,400058,400059,400060,400061,400062,400063,400064,400065,400066,400067,400068,400069,400070,400008,400014,400017,400058],
    ios:[100000,100001,100002,100003,100004,100005,100006,100007,100008,100009,100010,100011,100012,100013,100014,100015,100016,100017,100018,100019,100020,100021,100022,100023,100024,100025,100026,100027,100028,100029,100030,100031,100032,100033,100034,100035,100036,100037,100038,100039,100040,100041,100042,100043,100044,100045,100046,100047,100048,100049,100050,100051,100052,100053,100054,100055,100056,100057,100058,100059,100060,100061,100062,100063,100064,100065,100066,100067,100068,100069,100070,100071,100072,100073,100075,100076,100077,100078,100079]
  },
  country: 'JP',
  date: '2021-01-26',
  os: ['ios', 'google-play']
});