Custom module output help needed

Wappler General How To
,
#1

Hello Wapplers,

I’m trying to receive output from custom module based on JavaScript promise as an output (response).

exports.parse_website = async function (options) {
	options = this.parse(options);
	const axios = require('axios'); 
	const cheerio = require('cheerio'); 

	const extractLinks = $ => [ 
		...new Set( 
			$('.page-numbers a') 
				.map((_, a) => $(a).attr('href'))
				.toArray()
		), 
	]; 
	 
	axios.get(options.website_link).then(({ data }) => { 
		const $ = cheerio.load(data)
		const links = extractLinks($); 
		console.log(links);
	});
};

In Wappler WebServer Logs I see output:

  server-connect:router Serving serverConnect /api/fff +46s
  server-connect:app Executing action step parse_website +46s
  server-connect:app options: { website_link: 'https://scrapeme.live/shop/' } +0ms
[
  'https://scrapeme.live/shop/page/2/',
  'https://scrapeme.live/shop/page/3/',
  'https://scrapeme.live/shop/page/4/',
  'https://scrapeme.live/shop/page/46/',
  'https://scrapeme.live/shop/page/47/',
  'https://scrapeme.live/shop/page/48/'
]

I’ve tried to put return links; after console.log(links); but this return nothing because of a promise, not async function.

If I try to add return links; after promise - Wappler says links is not defined

#2

Try with:

let response = await axios.get(…)

And then use cheerio on the response.

1 Like
#3

In order to receive response code bellow is working:

exports.parse_website = async function (options) {
	options = this.parse(options);
	const axios = require('axios'); 
	const cheerio = require('cheerio'); 

	const extractLinks = $ => [ 
		...new Set( 
			$('.page-numbers a') 
				.map((_, a) => $(a).attr('href'))
				.toArray()
		), 
	]; 

	const sendGetRequest = async () => {
		try {
			const resp = await axios.get(options.website_link);
			const $ = cheerio.load(resp.data)
			const links = extractLinks($);
			return links;
		} catch (err) {
			console.error(err);
		}
	};

return sendGetRequest();

};
#4

Why do you need to wrap sendGetRequest in async function?

parse_website is already wrapped as async.

1 Like
#5

@JonL Thank you for pointing this out.

exports.parse_website = async function (options) {
	options = this.parse(options);
	const axios = require('axios'); 
	const cheerio = require('cheerio'); 
	const extractLinks = $ => [ 
		...new Set( 
			$('.page-numbers a') 
				.map((_, a) => $(a).attr('href'))
				.toArray()
		), 
	]; 
	
	const extractContent = $ => 
	$('.product') 
		.map((_, product) => { 
			const $product = $(product); 
			return { 
				id: $product.find('a[data-product_id]').attr('data-product_id'), 
				title: $product.find('h2').text(), 
				price: $product.find('.price').text(), 
			}; 
		}) 
		.toArray(); 

	const resp = await axios.get(options.website_link);
	const $ = cheerio.load(resp.data)
	const links = extractContent($);
	return links;
};
1 Like